diff --git a/.github/workflows/test-setup-rdma-runner.yaml b/.github/workflows/test-setup-rdma-runner.yaml new file mode 100644 index 0000000..f0527be --- /dev/null +++ b/.github/workflows/test-setup-rdma-runner.yaml @@ -0,0 +1,55 @@ +name: Test setup-rdma-runner + +on: + pull_request: + paths: + - 'setup-rdma-runner/**' + - '.github/workflows/test-setup-rdma-runner.yaml' + +jobs: + softroce: + name: SoftRoCE setup + runs-on: + - self-hosted + - rocky9 + - large + - gcloud + env: + RDMA_NETDEV: eth0 + RDMA_LINK: rxe-test + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up SoftRoCE + id: rdma + uses: ./setup-rdma-runner + with: + netdev: ${{ env.RDMA_NETDEV }} + link: ${{ env.RDMA_LINK }} + + - name: Check the reported address is the netdev's + run: | + EXPECTED=$(ip -4 addr show "$RDMA_NETDEV" | awk '/inet / {print $2}' | cut -d/ -f1) + test "${{ steps.rdma.outputs.ip }}" = "$EXPECTED" + + - name: Check the link carries RDMA traffic + run: | + ibv_rc_pingpong -d "$RDMA_LINK" -g 1 & + SERVER_PID=$! + sleep 1 + ibv_rc_pingpong -d "$RDMA_LINK" -g 1 "${{ steps.rdma.outputs.ip }}" + wait "$SERVER_PID" + + # Runners are reused between jobs, so a second setup must succeed over + # the link the first one left behind. + - name: Set up SoftRoCE over the existing link + uses: ./setup-rdma-runner + with: + netdev: ${{ env.RDMA_NETDEV }} + link: ${{ env.RDMA_LINK }} + + # The runner is shared, so the test link must not outlive the job. + - name: Delete the test link + if: always() + run: sudo rdma link delete "$RDMA_LINK" 2> /dev/null || true diff --git a/.github/workflows/test-start-miniring.yaml b/.github/workflows/test-start-miniring.yaml new file mode 100644 index 0000000..baafb86 --- /dev/null +++ b/.github/workflows/test-start-miniring.yaml @@ -0,0 +1,92 @@ +name: Test start-miniring + +on: + pull_request: + paths: + - 'start-miniring/**' + - '.github/workflows/test-start-miniring.yaml' + +jobs: + ready: + name: Readiness is awaited + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + # miniring is a release binary from another repository; a stub is enough + # to exercise the action's own behaviour. + - name: Install a miniring stub + run: | + cat > miniring <<'STUB' + #!/usr/bin/env bash + printf '[%s]\n' "$@" > argv.txt + echo "All processes started" + sleep 300 + STUB + chmod +x miniring + + - name: Start miniring + uses: ./start-miniring + with: + bin: ./miniring + nodes: 1 + disks: 10 + node-bin-path: binaries/bizstorenode + disk-bin-path: binaries/biziod + extra-args: | + --sproxyd.enabled + --sproxyd.port=10000 + --biziod.extra-args=-V debug + + # Make sure all the passed parameters were parsed properly. + - name: Check the counts and binary paths reached miniring + run: | + grep -qxF -- '[-n]' argv.txt + grep -qxF -- '[10]' argv.txt + grep -qxF -- '[binaries/bizstorenode]' argv.txt + + # An option whose value holds a space must stay one argument + # (eg: --biziod.extra-args=-V debug). + - name: Check the extra options kept their quoting + run: | + grep -qxF -- '[--sproxyd.port=10000]' argv.txt + grep -qxF -- '[--biziod.extra-args=-V debug]' argv.txt + + - name: Check the pid file points at a running process + run: kill -0 "$(cat miniring.pid)" + + - name: Stop miniring + if: always() + run: sudo kill "$(cat miniring.pid)" 2> /dev/null || true + + never-ready: + name: A cluster that never reports readiness fails the step + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install a stub that never reports readiness + run: | + printf '#!/usr/bin/env bash\necho booting\nsleep 300\n' > miniring + chmod +x miniring + + - name: Start miniring + id: start + continue-on-error: true + uses: ./start-miniring + with: + bin: ./miniring + nodes: 1 + disks: 1 + node-bin-path: binaries/bizstorenode + disk-bin-path: binaries/biziod + timeout: 5 + + - name: Check the step failed + run: test '${{ steps.start.outcome }}' = failure + + - name: Stop miniring + if: always() + run: sudo kill "$(cat miniring.pid)" 2> /dev/null || true diff --git a/README.md b/README.md index d6d1e4e..87bd8af 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,12 @@ Call different workflows running on defined hour with specific arguments. ## actions-docker-upload Bunch of step to upload an image speed up with `actions@cache`. + +## setup-rdma-runner +Set up SoftRoCE on a RHEL-family runner: install the RDMA userspace tools, +load `rdma_rxe`, attach a link to a netdev, and output the address to bind +RDMA endpoints to. + +## start-miniring +Spawn a miniring cluster in the background, wait until it reports readiness, +and record its pid. Stopping it is left to the caller. diff --git a/setup-rdma-runner/action.yaml b/setup-rdma-runner/action.yaml new file mode 100644 index 0000000..670e4aa --- /dev/null +++ b/setup-rdma-runner/action.yaml @@ -0,0 +1,27 @@ +name: "setup RDMA runner" +description: 'Set up a SoftRoCE device on a RHEL-family runner and report the address to bind RDMA endpoints to' + +inputs: + netdev: + description: Network device the SoftRoCE link is attached to. + default: eth0 + + link: + description: Name of the SoftRoCE link to create. + default: rxe0 + +outputs: + ip: + description: IPv4 address of the netdev, to bind RDMA endpoints to. + value: ${{ steps.setup.outputs.ip }} + +runs: + using: "composite" + steps: + - name: Set up SoftRoCE + id: setup + shell: bash + env: + NETDEV: ${{ inputs.netdev }} + LINK: ${{ inputs.link }} + run: ${GITHUB_ACTION_PATH}/setup-rdma-runner.sh diff --git a/setup-rdma-runner/setup-rdma-runner.sh b/setup-rdma-runner/setup-rdma-runner.sh new file mode 100755 index 0000000..893bf75 --- /dev/null +++ b/setup-rdma-runner/setup-rdma-runner.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Set up SoftRoCE on the runner and report the address to bind RDMA to. + +set -eu -o pipefail + +# Install RDMA libraries +sudo dnf -y install libibverbs-utils librdmacm + +# Load SoftRoCE kernel module +sudo modprobe rdma_rxe + +# Attach SoftRoCE device to $NETDEV +sudo rdma link delete "${LINK}" 2> /dev/null || true +sudo rdma link add "${LINK}" type rxe netdev "${NETDEV}" + +# Verify RDMA device +ibv_devinfo -d "${LINK}" + +# Report the address to bind RDMA endpoints to +IP=$(ip -4 addr show "${NETDEV}" | awk '/inet / {print $2}' | cut -d/ -f1) +if [ -z "${IP}" ]; then + echo "ERROR: ${NETDEV} has no IPv4 address" >&2 + exit 1 +fi +echo "ip=${IP}" >> "${GITHUB_OUTPUT}" diff --git a/start-miniring/action.yaml b/start-miniring/action.yaml new file mode 100644 index 0000000..fc251c5 --- /dev/null +++ b/start-miniring/action.yaml @@ -0,0 +1,58 @@ +name: "start miniring" +description: 'Spawn a miniring cluster in the background and wait until it is ready' + +inputs: + bin: + description: Path to the miniring binary. + required: true + + nodes: + description: Number of bizstorenode instances to spawn. + required: true + + disks: + description: Number of biziod instances to spawn. + required: true + + node-bin-path: + description: Path to the bizstorenode binary. + required: true + + disk-bin-path: + description: Path to the biziod binary. + required: true + + extra-args: + description: | + Further miniring options, one per line, in `--option=value` form. + Connector options belong here, for example `--sproxyd.enabled`. + default: "" + + log: + description: File collecting the combined output of every spawned process. + default: miniring.log + + pid: + description: File the miniring pid is written to, for teardown. + default: miniring.pid + + timeout: + description: Seconds to wait for miniring to report readiness. + default: "60" + +runs: + using: "composite" + steps: + - name: Spawn miniring + shell: bash + env: + MINIRING_BIN: ${{ inputs.bin }} + NODES: ${{ inputs.nodes }} + DISKS: ${{ inputs.disks }} + NODE_BIN_PATH: ${{ inputs.node-bin-path }} + DISK_BIN_PATH: ${{ inputs.disk-bin-path }} + EXTRA_ARGS: ${{ inputs.extra-args }} + LOG: ${{ inputs.log }} + PID_FILE: ${{ inputs.pid }} + TIMEOUT: ${{ inputs.timeout }} + run: ${GITHUB_ACTION_PATH}/start-miniring.sh diff --git a/start-miniring/start-miniring.sh b/start-miniring/start-miniring.sh new file mode 100755 index 0000000..ac15561 --- /dev/null +++ b/start-miniring/start-miniring.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Spawn a miniring cluster and wait until it reports readiness. + +set -eu -o pipefail + +READY='All processes started' + +# Split the extra options on lines, so a value holding a space stays one +# argument +mapfile -t EXTRA < <(printf '%s\n' "${EXTRA_ARGS}" | sed -e 's/[[:space:]]*$//' -e '/^$/d') + +# Start miniring $NODES nodes $DISKS disks and record its pid +sudo prlimit --memlock=unlimited \ + "${MINIRING_BIN}" spawn -n "${NODES}" -d "${DISKS}" \ + --node-bin-path "${NODE_BIN_PATH}" \ + --disk-bin-path "${DISK_BIN_PATH}" \ + "${EXTRA[@]}" > "${LOG}" 2>&1 & +echo $! > "${PID_FILE}" + +# Wait for readiness +for _ in $(seq 1 "${TIMEOUT}"); do + if grep -q "${READY}" "${LOG}" 2> /dev/null; then + exit 0 + fi + sleep 1 +done + +# Report the log on timeout +cat "${LOG}" >&2 +echo "ERROR: miniring did not start within ${TIMEOUT}s" >&2 +exit 1