RING-54773: reusable RDMA actions for the ring RDMA CI

.github/workflows/test-setup-rdma-runner.yaml

@@ -0,0 +1,55 @@
+name: Test setup-rdma-runner
+
+on:
+  pull_request:
+    paths:
+      - 'setup-rdma-runner/**'
+      - '.github/workflows/test-setup-rdma-runner.yaml'
+
+jobs:
+  softroce:
+    name: SoftRoCE setup
+    runs-on:
+      - self-hosted
+      - rocky9
+      - large
+      - gcloud
+    env:
+      RDMA_NETDEV: eth0
+      RDMA_LINK: rxe-test
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Set up SoftRoCE
+        id: rdma
+        uses: ./setup-rdma-runner
+        with:
+          netdev: ${{ env.RDMA_NETDEV }}
+          link: ${{ env.RDMA_LINK }}
+
+      - name: Check the reported address is the netdev's
+        run: |
+          EXPECTED=$(ip -4 addr show "$RDMA_NETDEV" | awk '/inet / {print $2}' | cut -d/ -f1)
+          test "${{ steps.rdma.outputs.ip }}" = "$EXPECTED"
+
+      - name: Check the link carries RDMA traffic
+        run: |
+          ibv_rc_pingpong -d "$RDMA_LINK" -g 1 &
+          SERVER_PID=$!
+          sleep 1
+          ibv_rc_pingpong -d "$RDMA_LINK" -g 1 "${{ steps.rdma.outputs.ip }}"
+          wait "$SERVER_PID"
+
+      # Runners are reused between jobs, so a second setup must succeed over
+      # the link the first one left behind.
+      - name: Set up SoftRoCE over the existing link
+        uses: ./setup-rdma-runner
+        with:
+          netdev: ${{ env.RDMA_NETDEV }}
+          link: ${{ env.RDMA_LINK }}
+
+      # The runner is shared, so the test link must not outlive the job.
+      - name: Delete the test link
+        if: always()
+        run: sudo rdma link delete "$RDMA_LINK" 2> /dev/null || true

.github/workflows/test-start-miniring.yaml

@@ -0,0 +1,92 @@
+name: Test start-miniring
+
+on:
+  pull_request:
+    paths:
+      - 'start-miniring/**'
+      - '.github/workflows/test-start-miniring.yaml'
+
+jobs:
+  ready:
+    name: Readiness is awaited
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      # miniring is a release binary from another repository; a stub is enough
+      # to exercise the action's own behaviour.
+      - name: Install a miniring stub
+        run: |
+          cat > miniring <<'STUB'
+          #!/usr/bin/env bash
+          printf '[%s]\n' "$@" > argv.txt
+          echo "All processes started"
+          sleep 300
+          STUB
+          chmod +x miniring
+
+      - name: Start miniring
+        uses: ./start-miniring
+        with:
+          bin: ./miniring
+          nodes: 1
+          disks: 10
+          node-bin-path: binaries/bizstorenode
+          disk-bin-path: binaries/biziod
+          extra-args: |
+            --sproxyd.enabled
+            --sproxyd.port=10000
+            --biziod.extra-args=-V debug
+
+      # Make sure all the passed parameters were parsed properly.
+      - name: Check the counts and binary paths reached miniring
+        run: |
+          grep -qxF -- '[-n]' argv.txt
+          grep -qxF -- '[10]' argv.txt
+          grep -qxF -- '[binaries/bizstorenode]' argv.txt
+
+      # An option whose value holds a space must stay one argument
+      # (eg: --biziod.extra-args=-V debug).
+      - name: Check the extra options kept their quoting
+        run: |
+          grep -qxF -- '[--sproxyd.port=10000]' argv.txt
+          grep -qxF -- '[--biziod.extra-args=-V debug]' argv.txt
+
+      - name: Check the pid file points at a running process
+        run: kill -0 "$(cat miniring.pid)"
+
+      - name: Stop miniring
+        if: always()
+        run: sudo kill "$(cat miniring.pid)" 2> /dev/null || true
+
+  never-ready:
+    name: A cluster that never reports readiness fails the step
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Install a stub that never reports readiness
+        run: |
+          printf '#!/usr/bin/env bash\necho booting\nsleep 300\n' > miniring
+          chmod +x miniring
+
+      - name: Start miniring
+        id: start
+        continue-on-error: true
+        uses: ./start-miniring
+        with:
+          bin: ./miniring
+          nodes: 1
+          disks: 1
+          node-bin-path: binaries/bizstorenode
+          disk-bin-path: binaries/biziod
+          timeout: 5
+
+      - name: Check the step failed
+        run: test '${{ steps.start.outcome }}' = failure
+
+      - name: Stop miniring
+        if: always()
+        run: sudo kill "$(cat miniring.pid)" 2> /dev/null || true

README.md

@@ -7,3 +7,12 @@ Call different workflows running on defined hour with specific arguments.
 
 ## actions-docker-upload
 Bunch of step to upload an image speed up with `actions@cache`.
+
+## setup-rdma-runner
+Set up SoftRoCE on a RHEL-family runner: install the RDMA userspace tools,
+load `rdma_rxe`, attach a link to a netdev, and output the address to bind
+RDMA endpoints to.
+
+## start-miniring
+Spawn a miniring cluster in the background, wait until it reports readiness,
+and record its pid. Stopping it is left to the caller.

setup-rdma-runner/action.yaml

@@ -0,0 +1,27 @@
+name: "setup RDMA runner"
+description: 'Set up a SoftRoCE device on a RHEL-family runner and report the address to bind RDMA endpoints to'
+
+inputs:
+  netdev:
+    description: Network device the SoftRoCE link is attached to.
+    default: eth0
+
+  link:
+    description: Name of the SoftRoCE link to create.
+    default: rxe0
+
+outputs:
+  ip:
+    description: IPv4 address of the netdev, to bind RDMA endpoints to.
+    value: ${{ steps.setup.outputs.ip }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up SoftRoCE
+      id: setup
+      shell: bash
+      env:
+        NETDEV: ${{ inputs.netdev }}
+        LINK: ${{ inputs.link }}
+      run: ${GITHUB_ACTION_PATH}/setup-rdma-runner.sh

setup-rdma-runner/setup-rdma-runner.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# Set up SoftRoCE on the runner and report the address to bind RDMA to.
+
+set -eu -o pipefail
+
+# Install RDMA libraries
+sudo dnf -y install libibverbs-utils librdmacm
+
+# Load SoftRoCE kernel module
+sudo modprobe rdma_rxe
+
+# Attach SoftRoCE device to $NETDEV
+sudo rdma link delete "${LINK}" 2> /dev/null || true
+sudo rdma link add "${LINK}" type rxe netdev "${NETDEV}"
+
+# Verify RDMA device
+ibv_devinfo -d "${LINK}"
+
+# Report the address to bind RDMA endpoints to
+IP=$(ip -4 addr show "${NETDEV}" | awk '/inet / {print $2}' | cut -d/ -f1)
+if [ -z "${IP}" ]; then
+    echo "ERROR: ${NETDEV} has no IPv4 address" >&2
+    exit 1
+fi
+echo "ip=${IP}" >> "${GITHUB_OUTPUT}"

start-miniring/action.yaml

@@ -0,0 +1,58 @@
+name: "start miniring"
+description: 'Spawn a miniring cluster in the background and wait until it is ready'
+
+inputs:
+  bin:
+    description: Path to the miniring binary.
+    required: true
+
+  nodes:
+    description: Number of bizstorenode instances to spawn.
+    required: true
+
+  disks:
+    description: Number of biziod instances to spawn.
+    required: true
+
+  node-bin-path:
+    description: Path to the bizstorenode binary.
+    required: true
+
+  disk-bin-path:
+    description: Path to the biziod binary.
+    required: true
+
+  extra-args:
+    description: |
+      Further miniring options, one per line, in `--option=value` form.
+      Connector options belong here, for example `--sproxyd.enabled`.
+    default: ""
+
+  log:
+    description: File collecting the combined output of every spawned process.
+    default: miniring.log
+
+  pid:
+    description: File the miniring pid is written to, for teardown.
+    default: miniring.pid
+
+  timeout:
+    description: Seconds to wait for miniring to report readiness.
+    default: "60"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Spawn miniring
+      shell: bash
+      env:
+        MINIRING_BIN: ${{ inputs.bin }}
+        NODES: ${{ inputs.nodes }}
+        DISKS: ${{ inputs.disks }}
+        NODE_BIN_PATH: ${{ inputs.node-bin-path }}
+        DISK_BIN_PATH: ${{ inputs.disk-bin-path }}
+        EXTRA_ARGS: ${{ inputs.extra-args }}
+        LOG: ${{ inputs.log }}
+        PID_FILE: ${{ inputs.pid }}
+        TIMEOUT: ${{ inputs.timeout }}
+      run: ${GITHUB_ACTION_PATH}/start-miniring.sh

start-miniring/start-miniring.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Spawn a miniring cluster and wait until it reports readiness.
+
+set -eu -o pipefail
+
+READY='All processes started'
+
+# Split the extra options on lines, so a value holding a space stays one
+# argument
+mapfile -t EXTRA < <(printf '%s\n' "${EXTRA_ARGS}" | sed -e 's/[[:space:]]*$//' -e '/^$/d')
+
+# Start miniring $NODES nodes $DISKS disks and record its pid
+sudo prlimit --memlock=unlimited \
+    "${MINIRING_BIN}" spawn -n "${NODES}" -d "${DISKS}" \
+    --node-bin-path "${NODE_BIN_PATH}" \
+    --disk-bin-path "${DISK_BIN_PATH}" \
+    "${EXTRA[@]}" > "${LOG}" 2>&1 &
+echo $! > "${PID_FILE}"
+
+# Wait for readiness
+for _ in $(seq 1 "${TIMEOUT}"); do
+    if grep -q "${READY}" "${LOG}" 2> /dev/null; then
+        exit 0
+    fi
+    sleep 1
+done
+
+# Report the log on timeout
+cat "${LOG}" >&2
+echo "ERROR: miniring did not start within ${TIMEOUT}s" >&2
+exit 1