From 06d0452d695ff98a312ad6df3acdac0b3e0406f9 Mon Sep 17 00:00:00 2001 From: Axelle Date: Mon, 21 Sep 2026 13:21:44 +0200 Subject: [PATCH 1/4] RING-54773: add setup-rdma-runner, a SoftRoCE setup action The RDMA jobs in scality/ring repeat this SoftRoCE setup three times and derive the bind address inline. The action reports that address as an output instead. --- .github/workflows/test-setup-rdma-runner.yaml | 52 +++++++++++++++++++ README.md | 6 +++ setup-rdma-runner/action.yaml | 27 ++++++++++ setup-rdma-runner/setup-rdma-runner.sh | 26 ++++++++++ 4 files changed, 111 insertions(+) create mode 100644 .github/workflows/test-setup-rdma-runner.yaml create mode 100644 setup-rdma-runner/action.yaml create mode 100755 setup-rdma-runner/setup-rdma-runner.sh diff --git a/.github/workflows/test-setup-rdma-runner.yaml b/.github/workflows/test-setup-rdma-runner.yaml new file mode 100644 index 0000000..7a12b79 --- /dev/null +++ b/.github/workflows/test-setup-rdma-runner.yaml @@ -0,0 +1,52 @@ +name: Test setup-rdma-runner + +on: + pull_request: + paths: + - 'setup-rdma-runner/**' + - '.github/workflows/test-setup-rdma-runner.yaml' + +jobs: + softroce: + name: SoftRoCE setup + runs-on: + - self-hosted + - rocky9 + - large + - gcloud + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up SoftRoCE + id: rdma + uses: ./setup-rdma-runner + with: + netdev: eth0 + link: rxe-test + + - name: Check the reported address is the netdev's + run: | + EXPECTED=$(ip -4 addr show eth0 | awk '/inet / {print $2}' | cut -d/ -f1) + test "${{ steps.rdma.outputs.ip }}" = "$EXPECTED" + + - name: Check the link carries RDMA traffic + run: | + ibv_rc_pingpong -d rxe-test -g 1 & + SERVER_PID=$! + sleep 1 + ibv_rc_pingpong -d rxe-test -g 1 "${{ steps.rdma.outputs.ip }}" + wait "$SERVER_PID" + + # Runners are reused between jobs, so a second setup must succeed over + # the link the first one left behind. + - name: Set up SoftRoCE over the existing link + uses: ./setup-rdma-runner + with: + netdev: eth0 + link: rxe-test + + # The runner is shared, so the test link must not outlive the job. + - name: Delete the test link + if: always() + run: sudo rdma link delete rxe-test 2> /dev/null || true diff --git a/README.md b/README.md index d6d1e4e..d422dc1 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,9 @@ Call different workflows running on defined hour with specific arguments. ## actions-docker-upload Bunch of step to upload an image speed up with `actions@cache`. + +## setup-rdma-runner +Set up SoftRoCE on a RHEL-family runner: install the RDMA userspace tools, +load `rdma_rxe`, attach a link to a netdev, and output the address to bind +RDMA endpoints to. + diff --git a/setup-rdma-runner/action.yaml b/setup-rdma-runner/action.yaml new file mode 100644 index 0000000..670e4aa --- /dev/null +++ b/setup-rdma-runner/action.yaml @@ -0,0 +1,27 @@ +name: "setup RDMA runner" +description: 'Set up a SoftRoCE device on a RHEL-family runner and report the address to bind RDMA endpoints to' + +inputs: + netdev: + description: Network device the SoftRoCE link is attached to. + default: eth0 + + link: + description: Name of the SoftRoCE link to create. + default: rxe0 + +outputs: + ip: + description: IPv4 address of the netdev, to bind RDMA endpoints to. + value: ${{ steps.setup.outputs.ip }} + +runs: + using: "composite" + steps: + - name: Set up SoftRoCE + id: setup + shell: bash + env: + NETDEV: ${{ inputs.netdev }} + LINK: ${{ inputs.link }} + run: ${GITHUB_ACTION_PATH}/setup-rdma-runner.sh diff --git a/setup-rdma-runner/setup-rdma-runner.sh b/setup-rdma-runner/setup-rdma-runner.sh new file mode 100755 index 0000000..893bf75 --- /dev/null +++ b/setup-rdma-runner/setup-rdma-runner.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Set up SoftRoCE on the runner and report the address to bind RDMA to. + +set -eu -o pipefail + +# Install RDMA libraries +sudo dnf -y install libibverbs-utils librdmacm + +# Load SoftRoCE kernel module +sudo modprobe rdma_rxe + +# Attach SoftRoCE device to $NETDEV +sudo rdma link delete "${LINK}" 2> /dev/null || true +sudo rdma link add "${LINK}" type rxe netdev "${NETDEV}" + +# Verify RDMA device +ibv_devinfo -d "${LINK}" + +# Report the address to bind RDMA endpoints to +IP=$(ip -4 addr show "${NETDEV}" | awk '/inet / {print $2}' | cut -d/ -f1) +if [ -z "${IP}" ]; then + echo "ERROR: ${NETDEV} has no IPv4 address" >&2 + exit 1 +fi +echo "ip=${IP}" >> "${GITHUB_OUTPUT}" From 3d22a993f7afe37aaed831057a1c8472dee81afd Mon Sep 17 00:00:00 2001 From: Axelle Date: Mon, 21 Sep 2026 13:21:51 +0200 Subject: [PATCH 2/4] RING-54773: add start-miniring, a cluster spawn-and-wait action Two RDMA jobs in scality/ring spawn a miniring the same way and poll its log for the same readiness line. Connector options pass through as one line each, so the sproxyd flags a caller needs stay in that caller's workflow. Stopping the cluster is left to the caller. --- .github/workflows/test-start-miniring.yaml | 92 ++++++++++++++++++++++ README.md | 3 + start-miniring/action.yaml | 58 ++++++++++++++ start-miniring/start-miniring.sh | 32 ++++++++ 4 files changed, 185 insertions(+) create mode 100644 .github/workflows/test-start-miniring.yaml create mode 100644 start-miniring/action.yaml create mode 100755 start-miniring/start-miniring.sh diff --git a/.github/workflows/test-start-miniring.yaml b/.github/workflows/test-start-miniring.yaml new file mode 100644 index 0000000..57c24aa --- /dev/null +++ b/.github/workflows/test-start-miniring.yaml @@ -0,0 +1,92 @@ +name: Test start-miniring + +on: + pull_request: + paths: + - 'start-miniring/**' + - '.github/workflows/test-start-miniring.yaml' + +jobs: + ready: + name: Readiness is awaited + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + # miniring is a release binary from another repository; a stub is enough + # to exercise the action's own behaviour. + - name: Install a miniring stub + run: | + cat > miniring <<'STUB' + #!/usr/bin/env bash + printf '[%s]\n' "$@" > argv.txt + echo "All processes started" + sleep 300 + STUB + chmod +x miniring + + - name: Start miniring + uses: ./start-miniring + with: + bin: ./miniring + nodes: 1 + disks: 10 + node-bin-path: binaries/bizstorenode + disk-bin-path: binaries/biziod + extra-args: | + --sproxyd.enabled + --sproxyd.port=10000 + --biziod.extra-args=-V debug + + # Make sure all the passed parameters were parsed properly. + - name: Check the counts and binary paths reached miniring + run: | + grep -qxF -- '[-n]' argv.txt + grep -qxF -- '[10]' argv.txt + grep -qxF -- '[binaries/bizstorenode]' argv.txt + + # An option whose value holds a space must stay one argument + # (eg: --biziod.extra-args=-V debug). + - name: Check the extra options kept their quoting + run: | + grep -qxF -- '[--sproxyd.port=10000]' argv.txt + grep -qxF -- '[--biziod.extra-args=-V debug]' argv.txt + + - name: Check the pid file points at a running process + run: kill -0 "$(cat miniring.pid)" + + - name: Stop miniring + if: always() + run: sudo kill "$(cat miniring.pid)" 2> /dev/null || true + + never-ready: + name: A cluster that never reports readiness fails the step + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install a stub that never reports readiness + run: | + printf '#!/usr/bin/env bash\necho booting\nsleep 300\n' > miniring + chmod +x miniring + + - name: Start miniring + id: start + continue-on-error: true + uses: ./start-miniring + with: + bin: ./miniring + nodes: 1 + disks: 1 + node-bin-path: binaries/bizstorenode + disk-bin-path: binaries/biziod + timeout: 5 + + - name: Check the step failed + run: test '${{ steps.start.outcome }}' = failure + + - name: Stop miniring + if: always() + run: sudo kill "$(cat miniring.pid)" 2> /dev/null || true diff --git a/README.md b/README.md index d422dc1..87bd8af 100644 --- a/README.md +++ b/README.md @@ -13,3 +13,6 @@ Set up SoftRoCE on a RHEL-family runner: install the RDMA userspace tools, load `rdma_rxe`, attach a link to a netdev, and output the address to bind RDMA endpoints to. +## start-miniring +Spawn a miniring cluster in the background, wait until it reports readiness, +and record its pid. Stopping it is left to the caller. diff --git a/start-miniring/action.yaml b/start-miniring/action.yaml new file mode 100644 index 0000000..fc251c5 --- /dev/null +++ b/start-miniring/action.yaml @@ -0,0 +1,58 @@ +name: "start miniring" +description: 'Spawn a miniring cluster in the background and wait until it is ready' + +inputs: + bin: + description: Path to the miniring binary. + required: true + + nodes: + description: Number of bizstorenode instances to spawn. + required: true + + disks: + description: Number of biziod instances to spawn. + required: true + + node-bin-path: + description: Path to the bizstorenode binary. + required: true + + disk-bin-path: + description: Path to the biziod binary. + required: true + + extra-args: + description: | + Further miniring options, one per line, in `--option=value` form. + Connector options belong here, for example `--sproxyd.enabled`. + default: "" + + log: + description: File collecting the combined output of every spawned process. + default: miniring.log + + pid: + description: File the miniring pid is written to, for teardown. + default: miniring.pid + + timeout: + description: Seconds to wait for miniring to report readiness. + default: "60" + +runs: + using: "composite" + steps: + - name: Spawn miniring + shell: bash + env: + MINIRING_BIN: ${{ inputs.bin }} + NODES: ${{ inputs.nodes }} + DISKS: ${{ inputs.disks }} + NODE_BIN_PATH: ${{ inputs.node-bin-path }} + DISK_BIN_PATH: ${{ inputs.disk-bin-path }} + EXTRA_ARGS: ${{ inputs.extra-args }} + LOG: ${{ inputs.log }} + PID_FILE: ${{ inputs.pid }} + TIMEOUT: ${{ inputs.timeout }} + run: ${GITHUB_ACTION_PATH}/start-miniring.sh diff --git a/start-miniring/start-miniring.sh b/start-miniring/start-miniring.sh new file mode 100755 index 0000000..ac15561 --- /dev/null +++ b/start-miniring/start-miniring.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Spawn a miniring cluster and wait until it reports readiness. + +set -eu -o pipefail + +READY='All processes started' + +# Split the extra options on lines, so a value holding a space stays one +# argument +mapfile -t EXTRA < <(printf '%s\n' "${EXTRA_ARGS}" | sed -e 's/[[:space:]]*$//' -e '/^$/d') + +# Start miniring $NODES nodes $DISKS disks and record its pid +sudo prlimit --memlock=unlimited \ + "${MINIRING_BIN}" spawn -n "${NODES}" -d "${DISKS}" \ + --node-bin-path "${NODE_BIN_PATH}" \ + --disk-bin-path "${DISK_BIN_PATH}" \ + "${EXTRA[@]}" > "${LOG}" 2>&1 & +echo $! > "${PID_FILE}" + +# Wait for readiness +for _ in $(seq 1 "${TIMEOUT}"); do + if grep -q "${READY}" "${LOG}" 2> /dev/null; then + exit 0 + fi + sleep 1 +done + +# Report the log on timeout +cat "${LOG}" >&2 +echo "ERROR: miniring did not start within ${TIMEOUT}s" >&2 +exit 1 From 207515109a3275c5cc53722d73d66ca2e44f7c60 Mon Sep 17 00:00:00 2001 From: Axelle Date: Tue, 22 Sep 2026 17:02:24 +0200 Subject: [PATCH 3/4] RING-54773: bump the test workflows to actions/checkout v7 --- .github/workflows/test-setup-rdma-runner.yaml | 2 +- .github/workflows/test-start-miniring.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-setup-rdma-runner.yaml b/.github/workflows/test-setup-rdma-runner.yaml index 7a12b79..23345a2 100644 --- a/.github/workflows/test-setup-rdma-runner.yaml +++ b/.github/workflows/test-setup-rdma-runner.yaml @@ -16,7 +16,7 @@ jobs: - gcloud steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up SoftRoCE id: rdma diff --git a/.github/workflows/test-start-miniring.yaml b/.github/workflows/test-start-miniring.yaml index 57c24aa..baafb86 100644 --- a/.github/workflows/test-start-miniring.yaml +++ b/.github/workflows/test-start-miniring.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 # miniring is a release binary from another repository; a stub is enough # to exercise the action's own behaviour. @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Install a stub that never reports readiness run: | From 6a67e44d05a837a8fd5761240497488a1de93431 Mon Sep 17 00:00:00 2001 From: Axelle Date: Wed, 23 Sep 2026 09:58:08 +0200 Subject: [PATCH 4/4] RING-54773: name the netdev and link once in the SoftRoCE test The link name appears in the two setup calls, both pingpong invocations and the teardown, and every one of them has to name the device the action created. --- .github/workflows/test-setup-rdma-runner.yaml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-setup-rdma-runner.yaml b/.github/workflows/test-setup-rdma-runner.yaml index 23345a2..f0527be 100644 --- a/.github/workflows/test-setup-rdma-runner.yaml +++ b/.github/workflows/test-setup-rdma-runner.yaml @@ -14,6 +14,9 @@ jobs: - rocky9 - large - gcloud + env: + RDMA_NETDEV: eth0 + RDMA_LINK: rxe-test steps: - name: Checkout uses: actions/checkout@v7 @@ -22,20 +25,20 @@ jobs: id: rdma uses: ./setup-rdma-runner with: - netdev: eth0 - link: rxe-test + netdev: ${{ env.RDMA_NETDEV }} + link: ${{ env.RDMA_LINK }} - name: Check the reported address is the netdev's run: | - EXPECTED=$(ip -4 addr show eth0 | awk '/inet / {print $2}' | cut -d/ -f1) + EXPECTED=$(ip -4 addr show "$RDMA_NETDEV" | awk '/inet / {print $2}' | cut -d/ -f1) test "${{ steps.rdma.outputs.ip }}" = "$EXPECTED" - name: Check the link carries RDMA traffic run: | - ibv_rc_pingpong -d rxe-test -g 1 & + ibv_rc_pingpong -d "$RDMA_LINK" -g 1 & SERVER_PID=$! sleep 1 - ibv_rc_pingpong -d rxe-test -g 1 "${{ steps.rdma.outputs.ip }}" + ibv_rc_pingpong -d "$RDMA_LINK" -g 1 "${{ steps.rdma.outputs.ip }}" wait "$SERVER_PID" # Runners are reused between jobs, so a second setup must succeed over @@ -43,10 +46,10 @@ jobs: - name: Set up SoftRoCE over the existing link uses: ./setup-rdma-runner with: - netdev: eth0 - link: rxe-test + netdev: ${{ env.RDMA_NETDEV }} + link: ${{ env.RDMA_LINK }} # The runner is shared, so the test link must not outlive the job. - name: Delete the test link if: always() - run: sudo rdma link delete rxe-test 2> /dev/null || true + run: sudo rdma link delete "$RDMA_LINK" 2> /dev/null || true