diff --git a/--gui b/--gui deleted file mode 100644 index e69de29bb..000000000 diff --git a/--num-robots b/--num-robots deleted file mode 100644 index e69de29bb..000000000 diff --git a/--sim b/--sim deleted file mode 100644 index e69de29bb..000000000 diff --git a/--stress-iterations b/--stress-iterations deleted file mode 100644 index e69de29bb..000000000 diff --git a/--trajectory-types b/--trajectory-types deleted file mode 100644 index e69de29bb..000000000 diff --git a/-v b/-v deleted file mode 100644 index fa52f4143..000000000 --- a/-v +++ /dev/null @@ -1,11 +0,0 @@ -access control disabled, clients can connect from any host -============================= test session starts ============================== -platform linux -- Python 3.12.13, pytest-9.0.3, pluggy-1.6.0 -- /usr/local/bin/python3.12 -cachedir: /tmp/.pytest_cache -rootdir: /home/pranavkumara/Desktop/AirStack/tests -configfile: pytest.ini -plugins: dependency-0.6.1, timeout-2.4.0 -collecting ... collected 0 items - -- generated xml file: /home/pranavkumara/Desktop/AirStack/tests/results/2026-05-28_14-14-04/results.xml - -============================ no tests ran in 0.00s ============================= diff --git a/.agents/skills/bump-version-and-release/SKILL.md b/.agents/skills/bump-version-and-release/SKILL.md index 3c056b774..18792a965 100644 --- a/.agents/skills/bump-version-and-release/SKILL.md +++ b/.agents/skills/bump-version-and-release/SKILL.md @@ -64,11 +64,13 @@ Three workflows in `.github/workflows/` interact with `VERSION`: - **Trigger:** push to `main` or `develop` whose changed paths include `.env`, **and** the `VERSION=` line in `.env` differs from the previous commit. Also runs on manual `workflow_dispatch`. - **Behavior on tag change:** 1. Runs on a self-hosted ephemeral GPU runner (`[self-hosted, airstack-ephemeral]`). - 2. `docker compose build` for profiles `desktop,isaac-sim,ms-airsim`. - 3. `docker compose push` to `${PROJECT_DOCKER_REGISTRY}` (set in `.env` — currently `airlab-docker.andrew.cmu.edu/airstack`). - 4. Keyless `cosign sign` of every pushed image digest via GitHub OIDC. - 5. `cosign verify` against the workflow's certificate identity. + 2. Plans per service via `.github/workflows/scripts/docker_image_plan.py` (content fingerprint vs previous versioned image label). + 3. **Unchanged image inputs** → registry retag of the previous `v${PREV}_…` digest to `v${VERSION}_…` and `cache_*` (no rebuild). + 4. **Changed inputs** (or missing/unlabeled previous image, or `force_rebuild=true`) → `docker compose build` / `push` for those services only, labeling the new digest with `org.airstack.content-fingerprint`. + 5. Keyless `cosign sign` of every published image digest via GitHub OIDC. + 6. `cosign verify` against the workflow's certificate identity. - **Skip behavior:** if the merge commit on `main`/`develop` does not actually change `VERSION=`, the build job is skipped (the check-changes job sets `tag-changed=false`). +- **Docs-only VERSION bumps:** still required by `check-version-increment`, but publish should retag rather than rebuild once fingerprints are on the previous images. First publish after this feature lands (or `force_rebuild=true`) must rebuild to write the labels. ### 3. `deploy_docs_from_release.yaml` — versioned docs @@ -76,7 +78,7 @@ Three workflows in `.github/workflows/` interact with `VERSION`: - **Behavior:** runs `mike deploy --push --update-aliases latest`, publishing the docs site under the release tag and pointing the `latest` alias at it. - Companion workflows publish unversioned docs from `main` (default alias `main`) and `develop` (alias `develop`). -So the full release path is: bump `VERSION` → PR → merge to `main`/`develop` (rebuild + push + sign) → cut a GitHub Release matching that VERSION (versioned docs go live). +So the full release path is: bump `VERSION` → PR → merge to `main`/`develop` (retag unchanged images and/or rebuild changed ones + push + sign) → cut a GitHub Release matching that VERSION (versioned docs go live). ## Choosing the Bump Type diff --git a/.agents/skills/run-system-tests/SKILL.md b/.agents/skills/run-system-tests/SKILL.md index f9b41b727..2605c5b7b 100644 --- a/.agents/skills/run-system-tests/SKILL.md +++ b/.agents/skills/run-system-tests/SKILL.md @@ -97,6 +97,7 @@ The `system-tests.yml` workflow's `Parse pytest args` step automatically prepend - `/pytest -m takeoff_hover_land` → effectively runs `-m "build_packages or takeoff_hover_land"` - `/pytest` (no marks) → pytest defaults (everything) - `/pytest -m build_docker` → unchanged (the build_docker tests rebuild from scratch anyway) +- `/pytest -m build_packages` → **pull-only** (retag `cache_*`, no `image-build`, no Isaac). Add `--no-image-build` on other marks to skip the bake. This guarantees that ROS 2 workspaces are built inside the containers before any launch/liveliness test tries to source them. If you intentionally want to skip `build_packages` (e.g. you trust the prebuilt images), include it explicitly: `-m "liveliness and not build_packages"` would work, but the simpler path is to run locally where the prepend logic doesn't apply. diff --git a/.airstack/modules/osmo.sh b/.airstack/modules/osmo.sh index 053decbee..66ac29e02 100755 --- a/.airstack/modules/osmo.sh +++ b/.airstack/modules/osmo.sh @@ -120,19 +120,47 @@ function _osmo_prompt { fi } -# osmo:setup — interactively register the three OSMO credentials AirStack -# needs (airlab-docker-registry, airlab-docker-login, airlab-nucleus). -# Idempotent — re-running rotates the credentials. +# Helper: _osmo_prompt variant that falls back to $3 on empty input. +function _osmo_prompt_default { + local var_name="$1" + local prompt_text="$2" + local default_val="$3" + local saved_stty="" + + if [ -t 0 ]; then + saved_stty="$(stty -g 2>/dev/null || true)" + if [ -n "$saved_stty" ]; then + trap 'stty "$saved_stty" 2>/dev/null; trap - INT' INT + stty -icanon 2>/dev/null + fi + fi + read -r -p "${prompt_text} [${default_val}]: " "$var_name" + if [ -n "$saved_stty" ]; then + stty "$saved_stty" 2>/dev/null + trap - INT + fi + + _osmo_trim "$var_name" + if [ -z "${!var_name}" ]; then + printf -v "$var_name" '%s' "$default_val" + fi +} + +# osmo:setup — interactively register the OSMO credentials AirStack needs +# (airlab-docker-registry, airlab-docker-login, airlab-nucleus, optional +# airlab-storage). Idempotent — re-running rotates the credentials. function cmd_osmo_setup { _osmo_check_cli || return 1 cat >&2 <<'EOF' -This sets up the three per-user OSMO credentials AirStack-on-OSMO needs: +This sets up the per-user OSMO credentials AirStack-on-OSMO needs: 1. airlab-docker-registry (REGISTRY) — for OSMO to pull the workspace image 2. airlab-docker-login (GENERIC) — for the inner dockerd to pull AirStack images 3. airlab-nucleus (GENERIC) — for Isaac Sim Nucleus access + 4. airlab-storage (GENERIC) — OPTIONAL: push mission results to the + airlab-storage NAS + auto-teardown You'll be asked for: @@ -140,6 +168,9 @@ You'll be asked for: - your AirLab Docker password (same as your Andrew password) - your Nucleus API token (https://airlab-nucleus.andrew.cmu.edu/omni/web3/ → right-click cloud → API Tokens). NOT your Andrew password. + - (optional) your airlab-storage password — the SEPARATE Samba/SFTP/rsync + password set at https://airlab-storage.andrew.cmu.edu:4001/#/forgot-pwd, + NOT your Andrew/Docker password. Values go directly to OSMO; nothing is written to disk locally. @@ -202,7 +233,33 @@ EOF "omni_server=${omni_server}" \ || { log_error "osmo credential set airlab-nucleus failed"; return 1; } - log_info "All three credentials registered. List them with: osmo credential list" + # Optional: airlab-storage NAS upload. Only the password is a secret; the + # destination path is set per-mission (nas_dest: in the mission spec). + local want_storage="" + read -r -p "Configure airlab-storage NAS upload (optional)? [y/N]: " want_storage + case "$want_storage" in + y|Y|yes|Yes) + local storage_user storage_pass storage_host + _osmo_prompt_default storage_user "airlab-storage username (Andrew ID)" "$andrew_id" + _osmo_prompt storage_pass "airlab-storage password (hidden; the SEPARATE Samba/SFTP/rsync password)" true || return 1 + _osmo_prompt_default storage_host "airlab-storage host" "airlab-storage.andrew.cmu.edu" + + log_info "Refreshing airlab-storage (GENERIC)..." + osmo credential delete airlab-storage >/dev/null 2>&1 || true + osmo credential set airlab-storage \ + --type GENERIC \ + --payload "username=${storage_user}" \ + "password=${storage_pass}" \ + "host=${storage_host}" \ + || { log_error "osmo credential set airlab-storage failed"; return 1; } + log_info "airlab-storage registered. Set 'nas_dest: /volume//...' in a mission spec to upload results + auto-teardown." + ;; + *) + log_info "Skipping airlab-storage — NAS upload stays disabled (missions keep-alive as before)." + ;; + esac + + log_info "Credentials registered. List them with: osmo credential list" log_info "Next: airstack osmo:up [--pool POOL]" } @@ -686,6 +743,360 @@ function cmd_osmo_foxglove { --connect-timeout "$OSMO_PF_TIMEOUT" } +# Parse a duration (90s / 30m / 8h / bare number = minutes) to seconds. +function _osmo_parse_duration { + local v="$1" + if [[ "$v" =~ ^([0-9]+)([smh]?)$ ]]; then + local n="${BASH_REMATCH[1]}" u="${BASH_REMATCH[2]}" + case "$u" in + s) echo "$n" ;; + h) echo $(( n * 3600 )) ;; + *) echo $(( n * 60 )) ;; # m or bare → minutes + esac + else + log_error "invalid duration '$v' — use e.g. 8h, 480m, 3600s, or a bare number (minutes)" + return 1 + fi +} + +function _osmo_hms { + printf '%dh%02dm' $(( $1 / 3600 )) $(( ($1 % 3600) / 60 )) +} + +# Block until the mission finishes (the workspace log prints "mission_runner +# exited") or cap_s elapses, then run osmo:fetch. Requires keep-alive so the +# pod is still up to fetch from. +function _osmo_wait_and_fetch { + local wf="$1" cap_s="$2" + local dest="${OSMO_AUTOFETCH_DEST:-./osmo-results}" + local poll="${OSMO_AUTOFETCH_POLL_S:-120}" + local start now elapsed snapshot status + start="$(date +%s)" + log_info "auto-fetch armed: polling every ${poll}s for completion (cap $(_osmo_hms "$cap_s")) → ${dest}" + log_info " keep this terminal open (or background with nohup/tmux) until the fetch runs." + while :; do + now="$(date +%s)"; elapsed=$(( now - start )) + status="$(osmo workflow query "$wf" 2>/dev/null | awk -F': +' '/^Status/ {print $2; exit}' | tr -d ' \r\n' || true)" + case "$status" in + PENDING|RUNNING|"") ;; + *) log_warn "workflow ${wf} is ${status} — ending wait, attempting fetch."; break ;; + esac + snapshot="$(timeout 25 osmo workflow logs "$wf" -t workspace -n 600 2>/dev/null || true)" + if printf '%s\n' "$snapshot" | grep -q "mission_runner exited"; then + log_info "mission complete (after $(_osmo_hms "$elapsed")) — fetching." + break + fi + if [ "$elapsed" -ge "$cap_s" ]; then + log_warn "auto-fetch cap reached ($(_osmo_hms "$cap_s")) without completion — fetching what's there." + break + fi + sleep "$poll" + done + AIRSTACK_OSMO_WF="$wf" cmd_osmo_fetch "$dest" +} + +# osmo:autofetch — attach the auto-fetch poller to an already-running mission +# (one submitted without --auto-fetch, or whose terminal was closed). Polls the +# workspace log for completion, then runs osmo:fetch. +# +# Usage: airstack osmo:autofetch [DUR] (DUR = max-wait cap, default 8h; uses +# the saved workflow id, or AIRSTACK_OSMO_WF to target a specific one.) +function cmd_osmo_autofetch { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + local cap_s + cap_s="$(_osmo_parse_duration "${1:-8h}")" || return 1 + _osmo_wait_and_fetch "$wf" "$cap_s" +} + +# osmo:mission — submit airstack-mission.yaml with a mission spec selected. +# +# Usage: airstack osmo:mission [--pool POOL] [--key PATH] +# [--branch BRANCH] [--no-keep-alive] +# [--auto-fetch DUR] [--nas-dest PATH] +# [--no-nas-upload] +# +# is a repo-relative path (e.g. osmo/missions/example_takeoff_land.yaml). +# The pod clones the branch and runs the mission spec from that clone, so the +# mission file must be committed and pushed. --no-keep-alive makes the task +# exit when the mission ends (frees the GPU, triggers the workflow's +# `outputs:` upload) instead of sleeping for `airstack osmo:fetch`. +# +# --auto-fetch DUR blocks after submit, polling the workspace log for mission +# completion and running osmo:fetch as soon as it's done (DUR — e.g. 8h, 480m — +# is the max wait before fetching anyway). Requires keep-alive (the default). +# +# If the airlab-storage credential is set (airstack osmo:setup) and the mission +# spec has `nas_dest:` (or --nas-dest PATH overrides it), the pod rsyncs results +# to the NAS then tears itself down. --no-nas-upload suppresses that. +function cmd_osmo_mission { + _osmo_check_cli || return 1 + + local mission="" + local pool="${OSMO_POOL:-}" + local pubkey_file="" + local branch="" + local branch_explicit=false + local keep_alive="true" + local auto_fetch_dur="" + local nas_dest="" + local no_nas_upload="false" + local extra_args=() + + while [ $# -gt 0 ]; do + case "$1" in + --pool) pool="$2"; shift 2 ;; + --key) pubkey_file="$2"; shift 2 ;; + --branch) branch="$2"; branch_explicit=true; shift 2 ;; + --no-keep-alive) keep_alive="false"; shift ;; + --auto-fetch) auto_fetch_dur="$2"; shift 2 ;; + --nas-dest) nas_dest="$2"; shift 2 ;; + --no-nas-upload) no_nas_upload="true"; shift ;; + -*) extra_args+=("$1"); shift ;; + *) + if [ -z "$mission" ]; then mission="$1"; else extra_args+=("$1"); fi + shift ;; + esac + done + + # Validate --auto-fetch up front (fail before submitting on a bad value). + local auto_fetch_cap_s="" + if [ -n "$auto_fetch_dur" ]; then + if [ "$keep_alive" != "true" ]; then + log_error "--auto-fetch requires the pod to stay alive; don't combine it with --no-keep-alive." + return 1 + fi + auto_fetch_cap_s="$(_osmo_parse_duration "$auto_fetch_dur")" || return 1 + fi + + # NAS auto-upload tears the pod down on completion, so --auto-fetch (which + # needs the pod alive) is moot when it runs. Best-effort heads-up. + if [ -n "$auto_fetch_dur" ] && [ "$no_nas_upload" != "true" ] \ + && osmo credential list 2>/dev/null | grep -q 'airlab-storage'; then + log_warn "airlab-storage is configured; a mission with nas_dest set uploads + tears down, making --auto-fetch moot." + fi + + if [ -z "$mission" ]; then + log_error "Usage: airstack osmo:mission [--pool POOL] [--branch BRANCH] [--no-keep-alive]" + log_error "Available missions:" + ls "${PROJECT_ROOT}/osmo/missions/"*.yaml 2>/dev/null \ + | sed "s|${PROJECT_ROOT}/| |" >&2 + return 1 + fi + # Normalize to a repo-relative path — that's what the pod resolves + # against its clone of the branch. + mission="${mission#"${PROJECT_ROOT}"/}" + if [ ! -f "${PROJECT_ROOT}/${mission}" ]; then + log_error "Mission file not found locally: ${PROJECT_ROOT}/${mission}" + return 1 + fi + + if [ -z "$pubkey_file" ]; then + if ! pubkey_file="$(_osmo_pick_pubkey)"; then + log_error "No SSH public key found in ~/.ssh. Generate one with: ssh-keygen -t ed25519" + return 1 + fi + fi + + local workflow_yaml="${PROJECT_ROOT}/osmo/workflows/airstack-mission.yaml" + if [ ! -f "$workflow_yaml" ]; then + log_error "Workflow file not found: ${workflow_yaml}" + return 1 + fi + + # The pod runs the mission file from its clone of origin/, so an + # unpushed mission spec is the most common "why is it running the wrong + # thing" failure — same auto-pin + pushed check as osmo:up. + if [ "$branch_explicit" = false ] && [ -z "$branch" ]; then + branch="$(_osmo_local_branch)" + if [ -n "$branch" ]; then + log_info "Auto-detected local branch '${branch}'; pod will clone from origin/${branch} (override with --branch main)." + fi + fi + if [ -n "$branch" ]; then + _osmo_check_branch_pushed "$branch" + fi + + local cmd=(osmo workflow submit "$workflow_yaml") + if [ -n "$pool" ]; then + cmd+=(--pool "$pool") + else + log_warn "No --pool provided and OSMO_POOL is unset; using your osmo profile's default pool." + fi + # Single --set-env: the flag is variadic and a second occurrence silently + # drops the first (see cmd_osmo_up). + local env_kvs=( + "SSH_PUB_KEY=$(cat "$pubkey_file")" + "OSMO_MISSION_FILE=${mission}" + "OSMO_MISSION_KEEP_ALIVE=${keep_alive}" + ) + if [ -n "$branch" ]; then + env_kvs+=("AIRSTACK_BRANCH=${branch}") + fi + if [ -n "$nas_dest" ]; then + env_kvs+=("OSMO_MISSION_UPLOAD_DEST=${nas_dest}") + fi + if [ "$no_nas_upload" = "true" ]; then + env_kvs+=("OSMO_MISSION_NO_UPLOAD=true") + fi + cmd+=(--set-env "${env_kvs[@]}") + if [ ${#extra_args[@]} -gt 0 ]; then + cmd+=("${extra_args[@]}") + fi + + log_info "Submitting mission '${mission}' (keep_alive=${keep_alive}): ${cmd[*]}" + local output + if ! output="$("${cmd[@]}" 2>&1)"; then + echo "$output" >&2 + log_error "osmo workflow submit failed." + return 1 + fi + echo "$output" + + local wf_id + wf_id="$(echo "$output" | awk -F'- ' '/^Workflow ID/ {print $2; exit}' | tr -d ' \r\n')" + if [ -z "$wf_id" ]; then + log_warn "Could not parse workflow id from submit output. Set it manually:" + log_warn " echo > ${OSMO_STATE_FILE}" + return 0 + fi + _osmo_save_wf_id "$wf_id" + + if [ -n "$auto_fetch_cap_s" ]; then + _osmo_wait_and_fetch "$wf_id" "$auto_fetch_cap_s" + return $? + fi + + log_info "Next steps:" + log_info " airstack osmo:logs # follow mission progress" + log_info " airstack osmo:fetch # download bags + results (keep-alive mode)" + log_info " airstack osmo:down # cancel when done (results die with the pod!)" + if [ -n "$nas_dest" ] && [ "$no_nas_upload" != "true" ]; then + log_info "NAS upload armed → results go to ${nas_dest}///, then the pod tears itself down." + fi +} + +# osmo:fetch — download mission results (mcap bags, logs, summaries) from the +# pod to the laptop over the authenticated ssh port-forward. +# +# Usage: airstack osmo:fetch [dest-dir] +# +# Incremental and resumable (rsync): safe to run mid-mission to pull finished +# iterations while the next one flies, and again later to top up. Falls back +# to scp -r (non-incremental) if rsync isn't installed. +# +# Note: the osmo CLI also ships `osmo workflow rsync download`, which could +# replace the port-forward + ssh below; we use the ssh path because the +# sshd + port-forward channel is already validated infrastructure for this +# workflow (osmo:ide) and works uniformly across osmo CLI versions. +function cmd_osmo_fetch { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + local dest="${1:-./osmo-results}" + local remote_path="/root/AirStack/osmo/results/" # trailing slash: rsync + # follows the symlink to + # /osmo/output/... on pods + local local_port="${OSMO_SSH_PORT%%:*}" + local ssh_opts=(-p "$local_port" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR) + + # Reuse an existing ssh port-forward (e.g. from osmo:ide) or spawn one + # for the duration of the fetch — same pattern as osmo:ide. + local pf_pid="" + if ! nc -z localhost "$local_port" 2>/dev/null; then + log_info "osmo workflow port-forward ${wf} workspace --port ${OSMO_SSH_PORT} (for the duration of the fetch)" + osmo workflow port-forward "$wf" workspace --port "$OSMO_SSH_PORT" --connect-timeout 600 \ + > "${OSMO_STATE_DIR}/fetch-pf.log" 2>&1 & + pf_pid=$! + trap '[ -n "'"$pf_pid"'" ] && kill "'"$pf_pid"'" 2>/dev/null; trap - EXIT INT TERM' EXIT INT TERM + local waited=0 + until nc -z localhost "$local_port" 2>/dev/null; do + sleep 1; waited=$((waited+1)) + if [ "$waited" -ge 30 ]; then + log_error "Timed out waiting for port-forward on :${local_port}. Log: ${OSMO_STATE_DIR}/fetch-pf.log" + return 1 + fi + if ! kill -0 "$pf_pid" 2>/dev/null; then + log_error "port-forward exited early. Tail:" + tail -10 "${OSMO_STATE_DIR}/fetch-pf.log" >&2 + return 1 + fi + done + fi + + mkdir -p "$dest" + log_info "Fetching ${remote_path} → ${dest}" + local rc + if command -v rsync >/dev/null 2>&1; then + rsync -az --partial --info=progress2 -e "ssh ${ssh_opts[*]}" \ + "root@localhost:${remote_path}" "$dest/" + rc=$? + else + log_warn "rsync not found; falling back to scp -r (non-incremental)." + scp "${ssh_opts[@]}" -r "root@localhost:${remote_path}." "$dest/" + rc=$? + fi + + if [ -n "$pf_pid" ]; then + kill "$pf_pid" 2>/dev/null + trap - EXIT INT TERM + fi + + if [ "$rc" -ne 0 ]; then + log_error "Fetch failed (exit ${rc}). Is the workflow still running, and has the mission produced results yet?" + return 1 + fi + log_info "Done. Open any .mcap under ${dest} directly in Foxglove (Open local file)." +} + +# osmo:stop — gracefully stop the running mission: SIGINT to mission_runner +# on the pod → current step aborts, recorders finalize their mcaps, bags + +# logs are collected, stack goes down. The pod itself stays alive +# (keep-alive), so follow with `airstack osmo:fetch` to download results. +function cmd_osmo_stop { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + local local_port="${OSMO_SSH_PORT%%:*}" + local ssh_opts=(-p "$local_port" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR) + + # Reuse an existing ssh port-forward or spawn one for this command — + # same pattern as osmo:fetch. + local pf_pid="" + if ! nc -z localhost "$local_port" 2>/dev/null; then + log_info "osmo workflow port-forward ${wf} workspace --port ${OSMO_SSH_PORT} (for the stop)" + osmo workflow port-forward "$wf" workspace --port "$OSMO_SSH_PORT" --connect-timeout 600 \ + > "${OSMO_STATE_DIR}/stop-pf.log" 2>&1 & + pf_pid=$! + local waited=0 + until nc -z localhost "$local_port" 2>/dev/null; do + sleep 1; waited=$((waited+1)) + if [ "$waited" -ge 30 ]; then + log_error "Timed out waiting for port-forward on :${local_port}. Log: ${OSMO_STATE_DIR}/stop-pf.log" + [ -n "$pf_pid" ] && kill "$pf_pid" 2>/dev/null + return 1 + fi + done + fi + + log_info "Sending stop signal to mission_runner on the pod..." + ssh "${ssh_opts[@]}" root@localhost \ + "pkill -INT -f 'osmo/workspace/mission_runner\.py' \ + && echo 'stop signal delivered' \ + || echo 'no running mission_runner found (mission already finished?)'" + local rc=$? + + if [ -n "$pf_pid" ]; then + kill "$pf_pid" 2>/dev/null + fi + if [ "$rc" -ne 0 ]; then + log_error "ssh to pod failed (exit ${rc})." + return 1 + fi + log_info "Mission stopping: current step aborts, recordings finalize, bags + logs are collected, stack goes down." + log_info "Watch progress: airstack osmo:logs Download results: airstack osmo:fetch" +} + # osmo:down — cancel the active workflow. Reminds you to push first. function cmd_osmo_down { _osmo_check_cli || return 1 @@ -693,6 +1104,7 @@ function cmd_osmo_down { log_warn "About to cancel workflow '${wf}'." log_warn "Anything not pushed to git in /root/AirStack inside the pod will be LOST." + log_warn "Mission results (bags/logs) on the pod are lost too — run 'airstack osmo:fetch' first." log_warn "Hit Ctrl-C in the next 5 seconds to abort." sleep 5 osmo workflow cancel "$wf" @@ -703,14 +1115,22 @@ function cmd_osmo_down { function register_osmo_commands { COMMANDS["osmo:setup"]="cmd_osmo_setup" COMMANDS["osmo:up"]="cmd_osmo_up" + COMMANDS["osmo:mission"]="cmd_osmo_mission" + COMMANDS["osmo:fetch"]="cmd_osmo_fetch" + COMMANDS["osmo:autofetch"]="cmd_osmo_autofetch" + COMMANDS["osmo:stop"]="cmd_osmo_stop" COMMANDS["osmo:logs"]="cmd_osmo_logs" COMMANDS["osmo:ide"]="cmd_osmo_ide" COMMANDS["osmo:webrtc"]="cmd_osmo_webrtc" COMMANDS["osmo:foxglove"]="cmd_osmo_foxglove" COMMANDS["osmo:down"]="cmd_osmo_down" - COMMAND_HELP["osmo:setup"]="One-time per-user OSMO credential setup (airlab-docker-registry, airlab-docker-login, airlab-nucleus)" + COMMAND_HELP["osmo:setup"]="One-time per-user OSMO credential setup (airlab-docker-registry, airlab-docker-login, airlab-nucleus, optional airlab-storage for NAS upload)" COMMAND_HELP["osmo:up"]="Submit osmo/workflows/airstack-dev.yaml with your SSH pubkey injected (--pool POOL, --key PATH, --branch BRANCH)" + COMMAND_HELP["osmo:mission"]="Submit a batch mission (osmo/missions/*.yaml): repeated up→fly→record→down cycles (--pool POOL, --branch BRANCH, --no-keep-alive, --auto-fetch DUR, --nas-dest PATH, --no-nas-upload)" + COMMAND_HELP["osmo:fetch"]="Download mission results (mcap bags, logs, summaries) from the pod over ssh — incremental, safe to run mid-mission (osmo:fetch [dest-dir])" + COMMAND_HELP["osmo:autofetch"]="Attach the auto-fetch poller to the running mission (saved id or AIRSTACK_OSMO_WF): poll the workspace log until done, then osmo:fetch (osmo:autofetch [DUR], default 8h)" + COMMAND_HELP["osmo:stop"]="Gracefully stop the running mission: abort current step, finalize mcaps, collect bags+logs, stack down (pod stays up for osmo:fetch)" COMMAND_HELP["osmo:logs"]="Follow the workspace task logs (osmo workflow logs -t workspace -n 500; OSMO_LOGS_TASK / OSMO_LOGS_TAIL override)" COMMAND_HELP["osmo:ide"]="Port-forward sshd (2200:22) and open VS Code/Cursor on Host airstack-osmo" COMMAND_HELP["osmo:webrtc"]="Port-forward Isaac Sim WebRTC ranges (TCP foreground + UDP background)" diff --git a/.env b/.env index 85280be19..c9527e1f8 100644 --- a/.env +++ b/.env @@ -12,7 +12,7 @@ PROJECT_NAME="airstack" # If you've run ./airstack.sh setup, then this will auto-generate from the git commit hash every time a change is made # to a Dockerfile or docker-compose.yaml file. Otherwise this can also be set explicitly to make a release version. # auto-generated from git commit hash -VERSION="0.19.0-alpha.7" +VERSION="0.19.0-alpha.9" # Choose "dev" or "prebuilt". "dev" is for mounted code that must be built live. "prebuilt" is for built ros_ws baked into the image DOCKER_IMAGE_BUILD_MODE="dev" # Where to push and pull images from. Can replace with your docker hub username if using docker hub. diff --git a/.github/orchestrator/README.md b/.github/orchestrator/README.md index c10da3383..f2df1ff22 100644 --- a/.github/orchestrator/README.md +++ b/.github/orchestrator/README.md @@ -1,131 +1,128 @@ -# AirStack CI Orchestrator +# AirStack CI Orchestrator (OSMO backend) -This describes how to use a self-hosted OpenStack VM to run GitHub Actions jobs on truly ephemeral workers. The orchestrator is a Python service that continuously polls GitHub for queued workflow jobs, spawns a fresh OpenStack instance for each one with a single-use JIT runner token, and reaps (deletes) the instance when the job completes. This allows us to run CI workloads on GPU-equipped VMs without sharing any state between runs or exposing long-lived credentials on the worker. +This describes how a small always-on orchestrator service runs GitHub Actions jobs on truly ephemeral GPU workers scheduled by [NVIDIA OSMO](https://nvidia.github.io/OSMO/). The orchestrator is a Python service that continuously polls GitHub for queued workflow jobs, submits a fresh **OSMO workflow** for each one (a single-use JIT runner in a privileged, GPU-enabled container), and reaps it when the job completes. Each CI job runs on a clean pod with no state shared between runs and no long-lived credentials on the worker. -The orchestrator VM is the only host that holds the GitHub PAT and the OpenStack credential; the workers are destroyed after a single job. +This is a drop-in replacement for the previous OpenStack-Nova backend. The GitHub side is unchanged — `system-tests.yml` still uses `runs-on: [self-hosted, airstack-ephemeral]`, the single-use JIT runner config, and the same-repo fork guard. Only the *spawn target* changed from "create a Nova VM" to "submit an OSMO workflow", so the one-job-per-worker, destroy-after semantics are identical: when the runner's `run.sh` exits after one job, the OSMO task completes and the pod is torn down. + +The orchestrator host is the only machine that holds the GitHub PAT and the OSMO service-account token; workers are destroyed after a single job. ## Architecture ``` ┌─────────────────────────────────────────────────────────────┐ -│ Orchestrator VM (airstack-ci-cd-orchestrator) │ +│ Orchestrator host (airstack-ci-cd-orchestrator, no GPU) │ │ │ │ airstack-orchestrator.service → orchestrator.py │ │ spawn loop (every 15s): │ │ • GET /repos//actions/runs?status=queued │ │ • POST /repos//actions/runners/generate-jitconfig│ -│ • openstack server create (image, flavor, user_data) │ -│ • record (job_id → server_id) in state.json │ +│ • osmo workflow submit runner-workflow.yaml --pool ... │ +│ • record (job_id → workflow_id) in state.json │ │ reap loop (every 30s): │ -│ • job completed → openstack server delete │ -│ • job age > N min → force delete (straggler) │ -│ • owned but not in state → orphan reap │ +│ • job completed → osmo workflow cancel (if live) │ +│ • job age > N min → osmo workflow cancel (straggler) │ +│ • our-named but not in state → orphan cancel │ │ │ │ /etc/airstack-orchestrator/ │ │ config.yaml │ │ github-pat │ -│ /home/orchestrator/.config/openstack/clouds.yaml │ +│ osmo-token (OSMO service-account token) │ │ /var/lib/airstack-orchestrator/state.json │ +│ /var/lib/airstack-orchestrator/.config/osmo (CLI session) │ └─────────┬─────────────────────────────────┬─────────────────┘ - │ Nova / Neutron API │ GitHub REST API + │ osmo CLI (submit/query/cancel) │ GitHub REST API ▼ ▼ ┌──────────────────────────────────┐ ┌──────────────────────┐ -│ Ephemeral worker (per job) │ │ GitHub Actions │ -│ Image: Ubuntu-24.04-GPU-Headless│ │ workflow_job queue │ -│ cloud-init: │ └──────────────────────┘ -│ install docker + nv toolkit │ -│ download GH runner │ +│ OSMO CI GPU pool (privileged) │ │ GitHub Actions │ +│ Ephemeral runner pod (per job): │ │ workflow_job queue │ +│ Image: airstack-ci-runner │ └──────────────────────┘ +│ start dockerd (DinD) │ │ run.sh --jitconfig │ -│ shutdown -h +1 │ +│ exit → task done → pod reaped │ └──────────────────────────────────┘ ``` Key properties: -- **Truly ephemeral**: every job runs on a clean VM. No Docker layer cache pollution, no leftover networks, no carry-over from prior runs. +- **Truly ephemeral**: every job runs on a clean pod. No Docker layer cache pollution, no leftover containers, no carry-over from prior runs. - **PAT isolation**: the GitHub PAT lives only on the orchestrator. Workers receive a single-use [JIT runner config](https://docs.github.com/en/rest/actions/self-hosted-runners?apiVersion=2022-11-28#create-configuration-for-a-just-in-time-runner-for-a-repository) — a base64 token bound to one runner registration, valid only for a short window. -- **Application-credential auth**: the orchestrator authenticates to OpenStack with an application credential (revocable, scoped, no password), not the user's `openrc.sh`. -- **Crash-safe reaping**: every server we spawn is tagged with `airstack-role=ephemeral-runner`. The reap loop force-deletes any owned server not present in `state.json`, so a crashed orchestrator can't leak instances. +- **Service-account auth**: the orchestrator authenticates to OSMO with a shared, non-personal [service-account token](https://nvidia.github.io/OSMO/main/deployment_guide/appendix/authentication/service_accounts.html) (the analog of the old OpenStack application credential). CI runs never route through an individual's account, so PRs don't consume anyone's personal GPU quota and nothing breaks when a person leaves. +- **Crash-safe reaping**: every workflow is named `gha-runner--`. The reap loop cancels any active workflow with that prefix not present in `state.json`, so a crashed orchestrator can't leak workflows. ## Prerequisites -- OpenStack instance already setup for the orchestrator VM. The orchestrator itself is lightweight and doesn't need a GPU. 1 vCPU, 2GB RAM, and 20GB disk is sufficient for the orchestrator service. Make sure you can ssh into it and that it has outbound internet access. -- An OpenStack flavor with GPU passthrough and enough disk to run Docker + the tests. The orchestrator spawns workers from this flavor, so it must have a GPU and sufficient disk (or `boot_volume_size_gb` must be set) to run the workloads. It's common for GPU flavors to have `disk=0`, which means they boot from an ephemeral disk — in that case, you must set `boot_volume_size_gb` to a value large enough for the OS + Docker images + test assets (e.g., 40GB). If your OpenStack setup supports it, you can also boot from a Cinder volume sourced from an image; in that case, pre-bake Docker and the NVIDIA toolkit into the image to speed up boot time. + +- **Orchestrator host** — a small always-on VM (no GPU). 1 vCPU, 2GB RAM, 20GB disk, outbound internet to `api.github.com` and your OSMO URL. This is the only long-lived piece; the GPU compute is ephemeral pods on OSMO. +- **An OSMO service account + dedicated CI pool.** Ask your OSMO admin to: + 1. Create a service account (e.g. `svc-airstack-ci`) and a long-lived access token — `osmo user create` + `osmo token set`. On IdP-backed deployments (e.g. auth tied to the CMU Andrew directory) this is a non-personal identity, so it survives people graduating/leaving. If policy forbids OSMO-native service accounts, use a *functional/departmental* identity, never a personal one. + 2. Grant that account a role whose policy allows `workflow:Create/Cancel/Query` **scoped to a dedicated CI GPU pool** (e.g. `pool/airstack-ci`) that has its own allocation, so CI doesn't contend with researchers' interactive jobs. + 3. **Enable "Privileged Mode Allowed"** on that pool's platform. The AirStack tests run `airstack up` (docker compose) inside the worker, which requires an inner Docker daemon → a privileged container. Without this, submissions are rejected. + 4. Confirm the API gateway (Envoy) accepts OSMO access tokens for the API (not only interactive IdP logins). +- **A prebaked runner image** pushed to a registry the pool can pull (see below). ## One-time setup -### 1. Create OpenStack application credential +### 1. Build & push the runner image -On your local workstation (not the orchestrator VM): +The worker image bakes in Docker CE + compose, the NVIDIA container toolkit, and the GitHub Actions runner (what cloud-init used to install at boot on the VM), so pod start is fast and the JIT token can't expire mid-bootstrap. ```bash -source ~/.airlabcloud/openrc.sh -openstack application credential create airstack-orchestrator \ - --description "AirStack CI orchestrator — spawns ephemeral test runners" +cd .github/orchestrator +./build-and-push.sh +# or manually: +# docker build -f runner.Dockerfile \ +# --build-arg RUNNER_VERSION=2.334.0 \ +# -t airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0 . +# docker push airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0 ``` -The output prints `id` and `secret`. Build a `clouds.yaml`: - -```yaml -clouds: - airstack: - auth_type: v3applicationcredential - auth: - auth_url: https://airlab-cloud.andrew.cmu.edu:5000/v3/ - application_credential_id: - application_credential_secret: - region_name: Airlab - interface: public - identity_api_version: 3 +No local Docker? Submit the one-shot OSMO builder (needs your Harbor creds in OSMO): + +```bash +osmo workflow submit .github/orchestrator/build-runner-on-osmo.yaml \ + --pool airstack --priority HIGH ``` -### 2. Stage credentials on the orchestrator VM +Set `runner_image: airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0` in `config.yaml` (step 4). Keep `RUNNER_VERSION` in sync with an [actions/runner release](https://github.com/actions/runner/releases). -```bash -# clouds.yaml: install for the orchestrator user (created in step 3) -scp clouds.yaml ubuntu@:/tmp/clouds.yaml +**Pool note (AirLab):** use the Keycloak-autosynced `airstack` pool (`privileged_allowed: true`). A hand-created `airstack-ci` pool is wiped by `synchronize_osmo_team_pools.py`. Ephemerality is per-job OSMO workflows, not a separate pool. + +### 2. Stage credentials on the orchestrator host +```bash # GitHub PAT: needs `Actions: read/write` and `Administration: read/write` # (fine-grained) or classic `repo` scope. -scp ~/.airlabcloud/airstack-github-pat.txt \ - ubuntu@:/tmp/github-pat +scp ~/airstack-github-pat.txt ubuntu@:/tmp/github-pat + +# OSMO service-account token (from `osmo token set`, provided by your admin). +scp ~/svc-airstack-ci-token.txt ubuntu@:/tmp/osmo-token ``` ### 3. Run setup.sh -On the orchestrator VM: +On the orchestrator host: ```bash git clone https://github.com/castacks/AirStack.git /tmp/airstack sudo bash /tmp/airstack/.github/orchestrator/setup.sh ``` -`setup.sh` creates the `orchestrator` system user, builds the Python venv, copies `orchestrator.py` and `cloud-init.yaml.j2` into `/opt/airstack-orchestrator/`, scaffolds `/etc/airstack-orchestrator/`, installs the systemd unit, and consumes `/tmp/github-pat`. - -You still need to put the `clouds.yaml` in place under the orchestrator user's home: - -```bash -sudo install -d -o orchestrator -g orchestrator -m 0700 \ - /home/orchestrator/.config/openstack -sudo install -o orchestrator -g orchestrator -m 0600 \ - /tmp/clouds.yaml /home/orchestrator/.config/openstack/clouds.yaml -sudo shred -u /tmp/clouds.yaml -``` +`setup.sh` creates the `orchestrator` system user, installs the `osmo` CLI, builds the Python venv, copies `orchestrator.py` and `runner-workflow.yaml.j2` into `/opt/airstack-orchestrator/`, scaffolds `/etc/airstack-orchestrator/`, installs the systemd unit, and consumes `/tmp/github-pat` and `/tmp/osmo-token`. ### 4. Fill in `/etc/airstack-orchestrator/config.yaml` -Edit the placeholders the example ships with: - | Field | What goes here | How to find it | |------|---------------|----------------| -| `flavor_name` | OpenStack flavor with GPU + enough disk | `openstack flavor list` | -| `network_name` | Network the workers attach to | `openstack network list` | -| `keypair_name` | SSH keypair for break-glass access | `openstack keypair list` | -| `security_group` | Outbound 443 must be allowed | `openstack security group list` | -| `availability_zone` | Optional AZ for the spawned instance; leave empty to let Nova pick | `openstack availability zone list` | -| `boot_volume_size_gb` | Set >0 if your flavor has `disk=0` (common for GPU flavors) — boots from a Cinder volume of this size sourced from `image_id`; leave 0 for direct image-boot | `openstack flavor show ` (check disk field) | -| `floating_ips` | Pre-allocated FIP pool, rotated through sequentially — each spawn picks the first free one. `max_concurrent` is capped at `len(pool)`. Leave empty to skip FIP attachment | `openstack floating ip list` | +| `osmo_url` | Your OSMO web service URL | from your OSMO admin | +| `pool` | Dedicated CI GPU pool | `osmo pool list` / the OSMO UI | +| `platform` | Optional hardware type within the pool; empty = pool default | `osmo pool list` | +| `runner_image` | Image from step 1 | the registry you pushed to | +| `cpu` / `gpu` / `memory` / `storage` | Resource request for the worker | size for full stack + sim | +| `privileged` | Must be `true` (docker compose inside the pod) | — | +| `priority` | `HIGH` \| `NORMAL` \| `LOW` | — | | `repo` | `owner/name` of the repo to poll | from GitHub URL | -| `runner_version` | Version tag from [actions/runner releases](https://github.com/actions/runner/releases) | check before each major upgrade | +| `runner_version` | Runner version baked into `runner_image` | matches step 1 | +| `max_concurrent` | Max simultaneous in-flight workflows | — | +| `max_job_minutes` | Straggler cancel ceiling | exceed the longest job | ### 5. Start the service @@ -134,7 +131,7 @@ sudo systemctl enable --now airstack-orchestrator.service journalctl -u airstack-orchestrator.service -f ``` -You should see `orchestrator started: repo=... labels=... max_concurrent=N` and then periodic poll activity. +You should see `orchestrator started (OSMO backend): repo=... pool=... max_concurrent=N`, an `osmo login succeeded` line, and then periodic poll activity. ## End-to-end verification @@ -142,147 +139,168 @@ You should see `orchestrator started: repo=... labels=... max_concurrent=N` and # Trigger a fast build-only run. gh workflow run system-tests.yml -f marks=build_docker -# Within ~30s, a server should appear: -openstack server list --metadata airstack-role=ephemeral-runner -# or if your OpenStack setup doesn't support metadata queries: -openstack server list --name '^ephemeral-' +# Within ~30s, a workflow should appear in the CI pool: +osmo workflow list --name gha-runner- --pool airstack-ci # Watch GitHub → Actions → Runners — the ephemeral runner should appear, # pick up the job, then disappear. -# Within ~30s of job completion, the server should be gone: -openstack server list --metadata airstack-role=ephemeral-runner -openstack server list --name '^ephemeral-' +# Within ~30s of job completion, the workflow should be terminal / gone from +# the active list: +osmo workflow list --name gha-runner- --pool airstack-ci --status RUNNING PENDING WAITING ``` ## Operational notes -- **State file**: `/var/lib/airstack-orchestrator/state.json` is the in-flight job tracker. Wiping it triggers an orphan sweep on the next reap iteration — owned servers will be force-deleted. Don't wipe it while jobs are mid-flight unless that's what you want. -- **Stuck instance**: any server older than `max_job_minutes` (default 90) is force-deleted regardless of GitHub job status. Bump this if liveliness/autonomy runs grow longer than ~75 minutes. +- **State file**: `/var/lib/airstack-orchestrator/state.json` is the in-flight job tracker (`job_id → workflow_id`). Wiping it triggers an orphan sweep on the next reap iteration — active `gha-runner-*` workflows will be cancelled. Don't wipe it while jobs are mid-flight unless that's what you want. +- **Straggler**: any workflow whose job has run longer than `max_job_minutes` (default 48h) is force-cancelled regardless of GitHub job status. +- **OSMO token rotation** (tokens expire — default 31 days): mint a new one and restart. + ```bash + # (admin) osmo token set svc-airstack-ci-token-2 --user svc-airstack-ci \ + # --roles osmo-user --expires-at 2027-12-31 + sudo install -o root -g orchestrator -m 0640 /tmp/osmo-token /etc/airstack-orchestrator/osmo-token + sudo systemctl restart airstack-orchestrator.service # re-runs `osmo login` + ``` - **PAT rotation**: `sudo install -o root -g orchestrator -m 0640 /tmp/new-pat /etc/airstack-orchestrator/github-pat && sudo systemctl restart airstack-orchestrator.service`. -- **Pause spawning** (e.g. for maintenance): `sudo systemctl stop airstack-orchestrator.service`. Already-spawned workers will still complete their jobs and self-shutdown; on restart, the reap loop deletes them. -- **Logs**: `journalctl -u airstack-orchestrator.service -f`. Cloud-init logs from individual workers are visible only via `openstack console log show ` while the worker is running. +- **Pause spawning** (e.g. for maintenance): `sudo systemctl stop airstack-orchestrator.service`. Already-submitted workers still complete their jobs; on restart, the reap loop cleans up. +- **Logs**: `journalctl -u airstack-orchestrator.service -f`. Per-worker logs come from `osmo workflow logs `. ## Debugging a failed job -When a GitHub workflow run fails or stalls, the failure can be in any of four places: the orchestrator (didn't spawn), cloud-init (didn't bootstrap), the GH Actions runner (didn't register or crashed), or the workflow steps themselves. Each has a different inspection path. +When a GitHub workflow run fails or stalls, the failure can be in one of four places: the orchestrator (didn't submit), the OSMO task (didn't schedule/pull), the GH Actions runner (didn't register or crashed), or the workflow steps themselves. Each has a different inspection path. -### 1. Find which worker ran the job +### 1. Find which workflow ran the job -`state.json` is the authoritative job ↔ server ↔ floating-IP map: +`state.json` is the authoritative job ↔ workflow map: ```bash -sudo jq -r '.jobs | to_entries[] | "\(.key)\t\(.value.server_id)\t\(.value.floating_ip)\t\(.value.runner_name)"' \ +sudo jq -r '.jobs | to_entries[] | "\(.key)\t\(.value.workflow_id)\t\(.value.workflow_name)"' \ /var/lib/airstack-orchestrator/state.json ``` -Pick the row for your failing `job_id` (visible in the GitHub Actions URL). Save the values: +Pick the row for your failing `job_id` (visible in the GitHub Actions URL): ```bash JOB_ID=73286176852 # from the GitHub UI -SERVER=$(sudo jq -r ".jobs[\"$JOB_ID\"].server_id" /var/lib/airstack-orchestrator/state.json) -FIP=$( sudo jq -r ".jobs[\"$JOB_ID\"].floating_ip" /var/lib/airstack-orchestrator/state.json) +WF=$(sudo jq -r ".jobs[\"$JOB_ID\"].workflow_id" /var/lib/airstack-orchestrator/state.json) ``` -If the job isn't in `state.json`, the orchestrator never spawned for it — see step 2 below. +If the job isn't in `state.json`, the orchestrator never submitted for it — see step 2. -### 2. Did the orchestrator spawn at all? +### 2. Did the orchestrator submit at all? ```bash sudo journalctl -u airstack-orchestrator.service --since "30 min ago" --no-pager ``` -What you want to see for a healthy spawn: +Healthy submit looks like: ```text -spawned server for job () -attached floating IP to server (job ) +submitted workflow for job () ``` -Common things that block a spawn (and how to spot them): +Common things that block a submit (and how to spot them): | Log line / symptom | What it means | Fix | |---|---|---| -| `find_queued_jobs failed: 401 ...` | PAT expired / wrong scope | Rotate the PAT (see Operational notes) | -| `spawn failed for job ...: Block Device Mapping is Invalid` | Flavor has `disk=0` and `boot_volume_size_gb` is 0 | Set `boot_volume_size_gb > 0` | -| `no free floating IP in pool` | All FIPs in `floating_ips` are already in use | Wait for an in-flight job to complete, or expand the pool | -| `floating_ips configured but not found` | Pool addresses don't exist in the project | Double-check `openstack floating ip list` | -| Job is queued in GitHub but no `spawned` log | Runner labels in the workflow's `runs-on` don't match `runner_labels` in config | Make them match | - -### 3. SSH into a running worker +| `find_queued_jobs failed: 401 ...` | GitHub PAT expired / wrong scope | Rotate the PAT | +| `osmo login failed ...` / `auth error` | OSMO token expired/invalid, or Envoy rejects access tokens | Rotate the OSMO token; confirm gateway accepts access tokens | +| `osmo workflow submit failed ... privileged` | Pool platform doesn't allow privileged | Ask admin to enable "Privileged Mode Allowed" on the CI pool | +| `osmo workflow submit failed ... pool` / permission | Service-account role lacks `workflow:Create` on the pool | Fix the role's pool-scoped policy | +| Job queued in GitHub but no `submitted` log | `runs-on` labels don't match `runner_labels` | Make them match | -If the worker is `ACTIVE`, the floating IP is attached and you can connect directly. The keypair was injected during spawn — use the matching private key: +### 3. Inspect the workflow / worker ```bash -ssh -i .pem ubuntu@"$FIP" -``` +# Status and scheduling detail. +osmo workflow query "$WF" --verbose -If your workstation can't reach the FIP subnet, jump through the orchestrator (which is on the same network): +# Scheduling / lifecycle events (image pull, start, evict, ...). +osmo workflow events "$WF" --task runner -```bash -ssh -J ubuntu@ -i .pem ubuntu@"$FIP" +# Combined stdout of the runner task — shows dockerd start, run.sh, and the job. +osmo workflow logs "$WF" --task runner +osmo workflow logs "$WF" --task runner --error # error stream +osmo workflow logs "$WF" --task runner -n 300 # last 300 lines ``` -### 4. SSH into a SHUTOFF worker +### 4. Break-glass shell into a running worker -Workers shut themselves down after `run.sh` exits (whether the job succeeded, failed, or the runner crashed). The orchestrator only deletes a server once GitHub reports the job `completed`, so a SHUTOFF worker is preserved while you debug. +If the workflow is still `RUNNING`, exec into the pod (replaces the old SSH-via-floating-IP path): ```bash -# Optional but safer — keep the orchestrator from reaping mid-session. -sudo systemctl stop airstack-orchestrator.service - -openstack server start "$SERVER" -# Wait ~30s, then SSH using the FIP from state.json. -ssh -i .pem ubuntu@"$FIP" +osmo workflow exec "$WF" runner # /bin/bash in the runner task ``` -When done, delete the worker manually and resume the orchestrator: +Once inside: ```bash -openstack server delete "$SERVER" -sudo jq "del(.jobs[\"$JOB_ID\"])" /var/lib/airstack-orchestrator/state.json \ - | sudo tee /var/lib/airstack-orchestrator/state.json.new >/dev/null -sudo mv /var/lib/airstack-orchestrator/state.json.new /var/lib/airstack-orchestrator/state.json -sudo systemctl start airstack-orchestrator.service +# GitHub Actions runner diagnostics. +ls -lt /home/runner/actions-runner/_diag/ +tail -300 /home/runner/actions-runner/_diag/Runner_*.log +tail -300 /home/runner/actions-runner/_diag/Worker_*.log + +# Inner Docker daemon (a frequent failure point for `airstack up`). +cat /var/log/dockerd.log +docker info 2>&1 | head +nvidia-smi ``` -### 5. What to read once you're on the worker +### 5. Common failure patterns at the worker -```bash -# Combined boot + cloud-init output. Most useful single file: shows every -# line our airstack-runner-bootstrap.sh printed, including run.sh's exit. -sudo less /var/log/cloud-init-output.log -sudo tail -300 /var/log/cloud-init-output.log - -# Cloud-init's structured log — quick way to surface errors. -sudo grep -E 'WARN|ERROR|FAIL' /var/log/cloud-init.log - -# GitHub Actions runner diagnostics. The Worker_*.log corresponds to the -# actual job execution; Runner_*.log covers registration and dispatch. -ls -lt /home/ubuntu/actions-runner/_diag/ -sudo tail -300 /home/ubuntu/actions-runner/_diag/Runner_*.log -sudo tail -300 /home/ubuntu/actions-runner/_diag/Worker_*.log - -# Sanity-check Docker came up cleanly — a frequent failure point. -sudo systemctl status docker -docker info 2>&1 | head +| Symptom in `osmo workflow logs` | Cause | Fix | +|---|---|---| +| `dockerd did not become ready` | Pod not privileged / DinD blocked | Enable privileged on the pool platform | +| `nvidia-smi unavailable` / no GPU | GPU not requested/passed, or toolkit missing | Check `gpu:` request, platform GPUs, privileged | +| `Could not connect to api.github.com` | Egress blocked from the pool | Allow outbound 443 from the CI pool | +| `Bad credentials` / `Invalid ... runnerEvent` | JIT config TTL elapsed before `run.sh` started | Prebake the image (already done) so start is fast | +| `Cannot connect to the Docker daemon` during tests | inner dockerd crashed | Read `/var/log/dockerd.log` via `osmo workflow exec` | +| Runner registered, then `pytest` failed | A normal test failure | Read the GitHub Actions log — the canonical view | +| `No space left on device` | `storage` too small for images + sim assets | Bump `storage` in `config.yaml` | +| `failed to solve: ... mount source: "overlay" ... err: invalid argument` | Docker data-root landed on the pod's overlay rootfs | See "Nested DinD and overlayfs" below | + +### Nested DinD and overlayfs + +The single most likely way to break every Docker build at once. The pod's root +filesystem is overlayfs, and **Linux refuses to use a directory on overlayfs as +an overlay `upperdir`** (it returns `EINVAL`). A dockerd whose data-root sits on +the pod rootfs looks healthy — `docker info` works, image pulls succeed, because +containerd unpacks layers with plain writes — and then every build step that +needs a real mount fails: + +```text +failed to solve: process "/bin/bash -c apt-get ... " did not complete successfully: +mount source: "overlay", +target: "/var/lib/docker/buildkit/containerd-overlayfs/cachemounts/buildkit1459786452", +fstype: overlay, ... err: invalid argument ``` -### 6. Console log fallback +That signature took out all four `build_docker` tests and all four +`build_packages` tests in one run, with each failure looking like an unrelated +`apt-get`/`WORKDIR` problem. -Some flavors on this cloud don't expose the serial console (`openstack console log show` returns *Guest does not have a console available*). For those, the SSH path above is the only option. Where it does work, the console log persists across SHUTOFF and is faster than restarting the VM: +[`runner-entrypoint.sh`](runner-entrypoint.sh) handles this before starting +dockerd. It picks a storage backend by **performing a real overlay mount** to +test each option rather than trusting the filesystem type, and falls back in +this order: + +| Order | Backend | Notes | +|---|---|---| +| 1 | Loopback ext4 image mounted at `/var/lib/docker` | Preferred. Real `overlay2`, self-contained, dies with the pod. Sparse, so it only consumes what Docker writes. Sized to free space on `/` minus 20 GiB, or `DOCKER_LOOP_SIZE_MB`. | +| 2 | A real filesystem already mounted in the pod | Kubernetes `emptyDir`/`hostPath`/PVC volumes live on the node disk, not the overlay rootfs. `/osmo/data/output` and `/osmo/data/socket` are skipped — the OSMO ctrl sidecar owns them. | +| 3 | `fuse-overlayfs` driver | Stacks where the kernel driver won't. Needs `/dev/fuse`. | +| 4 | `vfs` driver | Always works, copies the whole filesystem per layer. Too slow and too large for the sim images — **reaching this is a red flag**, not a working state. | + +The chosen backend is logged at startup, so confirm it in the job log before +debugging anything else: ```bash -openstack console log show "$SERVER" | tail -200 +osmo workflow logs "$WF" --task runner | grep -E 'runner-entrypoint|storage driver' ``` -### 7. Common failure patterns at the worker +Backends 3 and 4 also set `features.containerd-snapshotter: false`, because +`storage-driver` is only honoured by the classic image store. -| Symptom in `cloud-init-output.log` (near end) | Cause | Fix | -|---|---|---| -| `Could not connect to api.github.com` / DNS errors | Security group blocking egress, or no NAT for the network | Allow outbound 443; if behind NAT, ensure FIP networking covers egress | -| `Bad credentials` / `Invalid configuration ... runnerEvent` | JIT config TTL elapsed before `run.sh` started — bootstrap took too long | Pre-bake Docker + nvidia-container-toolkit into the image to shrink bootstrap | -| `nvidia-ctk: command not found` or NVIDIA driver mismatch | Image's driver doesn't match the toolkit version | Use a different image, or pin a compatible toolkit version | -| `apt-get update` fails | Image's apt sources are unreachable from this network | Check network/security-group; or pre-bake packages into the image | -| Runner registered, then `pytest` failed | A normal test failure | Read the GitHub Actions log — that's the canonical view of the workflow output | -| `No space left on device` | `boot_volume_size_gb` too small for Docker images + sim assets | Bump `boot_volume_size_gb` | +The one-shot [`build-runner-on-osmo.yaml`](build-runner-on-osmo.yaml) builder +sidesteps the same problem differently — `vfs` plus `DOCKER_BUILDKIT=0` — which +is fine there because it builds one small image. diff --git a/.github/orchestrator/airstack-orchestrator.service b/.github/orchestrator/airstack-orchestrator.service index 7123232eb..0c5c1843d 100644 --- a/.github/orchestrator/airstack-orchestrator.service +++ b/.github/orchestrator/airstack-orchestrator.service @@ -1,5 +1,5 @@ [Unit] -Description=AirStack CI Orchestrator (spawns ephemeral OpenStack runners) +Description=AirStack CI Orchestrator (submits ephemeral OSMO runner workflows) Documentation=https://github.com/castacks/AirStack/tree/main/.github/orchestrator After=network-online.target Wants=network-online.target @@ -10,17 +10,20 @@ User=orchestrator Group=orchestrator WorkingDirectory=/opt/airstack-orchestrator -# Application credential lives in the orchestrator user's home so openstacksdk -# finds it via the default cloud-config search path. -Environment=HOME=/home/orchestrator -Environment=OS_CLIENT_CONFIG_FILE=/home/orchestrator/.config/openstack/clouds.yaml +# The `osmo` CLI persists its login session under $HOME/XDG dirs. Point them at +# the (writable) state dir so ProtectHome can stay read-only. setup.sh creates +# these directories owned by the orchestrator user. +Environment=HOME=/var/lib/airstack-orchestrator +Environment=XDG_CONFIG_HOME=/var/lib/airstack-orchestrator/.config +Environment=XDG_CACHE_HOME=/var/lib/airstack-orchestrator/.cache +Environment=XDG_STATE_HOME=/var/lib/airstack-orchestrator/.state ExecStart=/opt/airstack-orchestrator/venv/bin/python \ /opt/airstack-orchestrator/orchestrator.py \ --config /etc/airstack-orchestrator/config.yaml \ --pat /etc/airstack-orchestrator/github-pat \ --state /var/lib/airstack-orchestrator/state.json \ - --template /opt/airstack-orchestrator/cloud-init.yaml.j2 + --template /opt/airstack-orchestrator/runner-workflow.yaml.j2 Restart=always RestartSec=10 @@ -33,6 +36,8 @@ KillSignal=SIGTERM NoNewPrivileges=true ProtectSystem=strict ProtectHome=read-only +# ReadWritePaths re-grants write access under ProtectSystem/ProtectHome so the +# OSMO CLI session cache and state.json can be written. ReadWritePaths=/var/lib/airstack-orchestrator PrivateTmp=true diff --git a/.github/orchestrator/build-and-push.sh b/.github/orchestrator/build-and-push.sh new file mode 100755 index 000000000..aacefb002 --- /dev/null +++ b/.github/orchestrator/build-and-push.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Build & push the AirStack CI ephemeral-runner image to AirLab Harbor. +# Run on a linux/amd64 machine (or buildx --platform linux/amd64) with: +# docker login airlab-docker.andrew.cmu.edu +# +# Usage: +# ./build-and-push.sh +# RUNNER_VERSION=2.336.0 ./build-and-push.sh + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY="${REGISTRY:-airlab-docker.andrew.cmu.edu/airstack}" +RUNNER_VERSION="${RUNNER_VERSION:-2.336.0}" +IMAGE="${REGISTRY}/airstack-ci-runner:${RUNNER_VERSION}" + +echo "==> Building ${IMAGE}" +docker build \ + -f "${ROOT}/runner.Dockerfile" \ + --build-arg "RUNNER_VERSION=${RUNNER_VERSION}" \ + -t "${IMAGE}" \ + "${ROOT}" + +echo "==> Pushing ${IMAGE}" +docker push "${IMAGE}" + +echo "==> Done. Set in /etc/airstack-orchestrator/config.yaml:" +echo " runner_image: \"${IMAGE}\"" diff --git a/.github/orchestrator/build-runner-on-osmo.yaml b/.github/orchestrator/build-runner-on-osmo.yaml new file mode 100644 index 000000000..a44f21ab5 --- /dev/null +++ b/.github/orchestrator/build-runner-on-osmo.yaml @@ -0,0 +1,63 @@ +# One-shot OSMO job: build + push airstack-ci-runner to AirLab Harbor. +# Uses the existing privileged DinD workspace image (same as airstack-dev). +# +# Prereq: your OSMO profile has airlab-docker-login (+ auto REGISTRY cred). +# +# osmo workflow submit .github/orchestrator/build-runner-on-osmo.yaml \ +# --pool airstack --priority HIGH +# +# Watch: +# osmo workflow logs --task build +# Cancel when done if it hangs: +# osmo workflow cancel --force + +workflow: + name: build-airstack-ci-runner + resources: + build: + cpu: 8 + gpu: 0 + memory: 16Gi + storage: 100Gi + platform: default + tasks: + - name: build + image: airlab-docker.andrew.cmu.edu/airstack/airstack-osmo-workspace:latest + resource: build + privileged: true + credentials: + airlab-docker-login: + AIRLAB_REGISTRY_USER: username + AIRLAB_REGISTRY_PASS: password + environment: + AIRSTACK_REPO_URL: "https://github.com/castacks/AirStack.git" + AIRSTACK_BRANCH: "ci/osmo-orchestrator" + RUNNER_VERSION: "2.336.0" + REGISTRY: "airlab-docker.andrew.cmu.edu/airstack" + command: + - bash + - -lc + - | + set -euo pipefail + # Nested DinD: overlay-on-overlay breaks BuildKit. Use vfs + legacy builder. + mkdir -p /etc/docker + cat > /etc/docker/daemon.json <<'JSON' + {"storage-driver": "vfs"} + JSON + dockerd >/var/log/dockerd.log 2>&1 & + for _ in $(seq 1 90); do docker info >/dev/null 2>&1 && break; sleep 1; done + docker info >/dev/null 2>&1 || { cat /var/log/dockerd.log; exit 1; } + docker info | grep -i 'Storage Driver' || true + + echo "$AIRLAB_REGISTRY_PASS" | docker login airlab-docker.andrew.cmu.edu \ + -u "$AIRLAB_REGISTRY_USER" --password-stdin + + rm -rf /tmp/AirStack + git clone --depth 1 --branch "$AIRSTACK_BRANCH" "$AIRSTACK_REPO_URL" /tmp/AirStack + cd /tmp/AirStack/.github/orchestrator + IMAGE="${REGISTRY}/airstack-ci-runner:${RUNNER_VERSION}" + DOCKER_BUILDKIT=0 docker build -f runner.Dockerfile \ + --build-arg "RUNNER_VERSION=${RUNNER_VERSION}" \ + -t "$IMAGE" . + docker push "$IMAGE" + echo "PUSHED $IMAGE" diff --git a/.github/orchestrator/cloud-init.yaml.j2 b/.github/orchestrator/cloud-init.yaml.j2 deleted file mode 100644 index 921417c18..000000000 --- a/.github/orchestrator/cloud-init.yaml.j2 +++ /dev/null @@ -1,71 +0,0 @@ -#cloud-config -# Rendered per-spawn by orchestrator.py with two Jinja variables: -# encoded_jit_config - single-use base64 JIT config from GitHub -# runner_version - GitHub Actions runner version (e.g. 2.334.0) -# -# The base image (Ubuntu-24.04-GPU-Headless) already has NVIDIA drivers. -# This cloud-init adds Docker (with the compose plugin), nvidia-container-toolkit, -# downloads the GitHub Actions runner, registers it with the JIT config, runs -# exactly one job (the JIT config + --ephemeral makes the runner exit after one -# job), and shuts the VM down. The orchestrator then deletes the server. - -package_update: true -package_upgrade: false -packages: - - jq - - curl - - ca-certificates - - gnupg - -write_files: - - path: /usr/local/bin/airstack-runner-bootstrap.sh - permissions: "0755" - owner: root:root - content: | - #!/usr/bin/env bash - set -euxo pipefail - - # Install Docker (with compose plugin) from Docker's official channel. - # get.docker.com handles apt repo setup + nvidia-container-toolkit-compatible - # docker-ce, plus the docker-compose-plugin we need for `airstack up`. - curl -fsSL https://get.docker.com | sh - - # nvidia-container-toolkit is required for GPU containers (liveliness / - # autonomy tests). The base image has the NVIDIA *drivers* but we still - # need the container runtime hooks here. - distribution=$(. /etc/os-release; echo "$ID$VERSION_ID") - curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ - | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg - curl -fsSL "https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list" \ - | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ - > /etc/apt/sources.list.d/nvidia-container-toolkit.list - apt-get update - apt-get install -y nvidia-container-toolkit - nvidia-ctk runtime configure --runtime=docker - systemctl restart docker - - usermod -aG docker ubuntu - - # GitHub Actions runner. - RUNNER_VERSION="{{ runner_version }}" - RUNNER_DIR=/home/ubuntu/actions-runner - mkdir -p "$RUNNER_DIR" - cd "$RUNNER_DIR" - curl -fsSL -o runner.tar.gz \ - "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" - tar xzf runner.tar.gz - rm runner.tar.gz - chown -R ubuntu:ubuntu "$RUNNER_DIR" - - # Run exactly one job under the ubuntu user. The JIT config is single-use - # and ephemeral, so run.sh exits after one job completes. - sudo -u ubuntu --preserve-env=HOME -H bash -c \ - "cd '$RUNNER_DIR' && ./run.sh --jitconfig '{{ encoded_jit_config }}'" \ - || echo "runner exited non-zero (job failure or runner error)" - - # Backstop: power down. The orchestrator's reap loop is the authoritative - # deleter — it sees the GitHub job complete and calls Nova delete. - shutdown -h +1 - -runcmd: - - /usr/local/bin/airstack-runner-bootstrap.sh diff --git a/.github/orchestrator/config.example.yaml b/.github/orchestrator/config.example.yaml index 4a47bbe1f..ba1c64384 100644 --- a/.github/orchestrator/config.example.yaml +++ b/.github/orchestrator/config.example.yaml @@ -1,54 +1,66 @@ -# AirStack CI orchestrator configuration. +# AirStack CI orchestrator configuration (OSMO backend). # Copy to /etc/airstack-orchestrator/config.yaml and fill in placeholders. -# --- OpenStack target --- - -# Cloud profile name in ~/.config/openstack/clouds.yaml. -openstack_cloud: airstack - -# Ubuntu-24.04-Desktop (confirmed available on airlab-cloud). -image_id: 2ebb9061-8995-4238-a3cc-e230a3e863aa - -# OpenStack flavor with GPU + enough disk for Docker + sim images. -# Look up with: openstack flavor list -flavor_name: "gpu.rtxpro5000.1" - -# OpenStack network the ephemeral instance attaches to. Must allow outbound -# 443 to api.github.com (no inbound is required: the runner makes an outbound -# long-poll connection to GitHub). -network_name: "airstack.AirLab.Apps_group_network_gates" - -# OpenStack keypair injected into the instance for break-glass SSH access. -# The orchestrator never SSHes into workers itself. -keypair_name: "airstack-ci-cd" - -# Security group applied to spawned instances. Outbound 443 must be allowed. -security_group: "default" - -# OpenStack availability zone to spawn instances in (e.g. nova, gpu-zone-1). -# Leave empty to let Nova pick. -availability_zone: "gates" - -# If the chosen flavor has disk=0 (common for GPU flavors), Nova rejects -# direct image-boot with "Block Device Mapping is Invalid: You specified more -# local devices than the limit allows". Set this to >0 to boot from a Cinder -# volume of that size sourced from image_id (deleted on termination). Leave -# at 0 to boot directly from the image (only works for non-zero-disk flavors). -boot_volume_size_gb: 300 - -# Pre-allocated pool of floating IPs to rotate through for SSH access to -# workers. The orchestrator picks the first free IP from this list, in order, -# for each new spawn. When the worker is destroyed the IP auto-disassociates -# and returns to the pool. If non-empty, max_concurrent is capped at len(pool) -# so the orchestrator never spawns a worker it can't address. -# Allocate via: openstack floating ip create -# Leave empty to skip floating-IP attachment entirely. -floating_ips: [] -# Example: -# floating_ips: -# - 172.19.220.131 -# - 172.19.220.171 -# - 172.19.220.89 +# --- OSMO target --- + +# Path to the `osmo` CLI. The install script (see setup.sh / README) puts it on +# PATH as `osmo`; override with a full path if needed. +osmo_bin: "osmo" + +# URL of your OSMO web service (the control plane the CLI logs into). +# AirLab: https://airlab-share-01.andrew.cmu.edu +osmo_url: "https://airlab-share-01.andrew.cmu.edu" + +# File containing the OSMO service-account access token. This is the shared, +# non-personal "lab" identity — the analog of the old OpenStack application +# credential. The orchestrator runs `osmo login --method token --token-file` +# with it. Created by an OSMO admin via `osmo user create` + `osmo token set` +# (see README). It never leaves this host. +osmo_token_file: "/etc/airstack-orchestrator/osmo-token" + +# GPU pool for ephemeral runners. AirLab team pools are Keycloak-autosynced; +# use the stable `airstack` pool (privileged_allowed=true). A hand-made +# `airstack-ci` pool will be wiped by synchronize_osmo_team_pools.py unless +# it is added to Keycloak. The service-account needs workflow:Create on this +# pool (role osmo-airstack) plus osmo-user for cancel/query. +pool: "airstack" + +# Optional platform (hardware type) to target within the pool. Leave empty to +# use the pool's default platform. List options with `osmo pool list` / the UI. +platform: "default" + +# Scheduling priority: HIGH | NORMAL | LOW. +priority: "NORMAL" + +# --- Runner task (the per-job worker) --- + +# Prebaked image: Docker CE + compose + nvidia-container-toolkit + GH Actions +# runner. Build & push runner.Dockerfile to a registry the pool can pull from. +runner_image: "airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.336.0" + +# Resource request for the runner container. Size for the full stack build + +# sim (Isaac Sim / ms-airsim + robot + gcs) running under docker compose. +cpu: 8 +gpu: 1 +memory: "32Gi" +storage: "300Gi" + +# REQUIRED for the AirStack tests: they run `airstack up` (docker compose) +# inside the pod, which needs an inner Docker daemon -> a privileged container. +# The pool's platform must have "Privileged Mode Allowed" enabled by your OSMO +# admin, otherwise submission/scheduling will be rejected. +privileged: true + +# Use the node's host network for the runner container. Usually not needed +# (the runner only makes outbound calls to GitHub); leave false unless the +# tests need host networking. +host_network: false + +# GitHub Actions runner version to bake into runner_image. Kept here for +# reference/traceability; it is a build arg of runner.Dockerfile, not consumed +# by the orchestrator at runtime. Must match a tag at +# https://github.com/actions/runner/releases +runner_version: "2.336.0" # --- GitHub --- @@ -56,23 +68,22 @@ floating_ips: [] repo: "castacks/AirStack" # Labels the orchestrator polls for. A queued workflow_job whose `labels` -# array is a superset of this list gets a server spawned for it. +# array is a superset of this list gets a workflow submitted for it. These are +# unchanged from the OpenStack backend, so system-tests.yml needs no edits. runner_labels: - self-hosted - airstack-ephemeral -# GitHub Actions runner version (must exist as a release tag at -# https://github.com/actions/runner/releases). -runner_version: "2.334.0" - # --- Limits --- -# Maximum simultaneous in-flight ephemeral instances. +# Maximum simultaneous in-flight workflows the orchestrator will submit. OSMO +# queues anything beyond the pool's capacity on its own, but this caps how many +# jobs we hand it at once. max_concurrent: 3 -# Hard ceiling for a single job. Past this age the reaper force-deletes the -# server even if GitHub still reports the job as in-progress. Must comfortably -# exceed the longest expected job (autonomy/liveliness runs). +# Hard ceiling for a single job. Past this age the reaper cancels the workflow +# even if GitHub still reports the job as in-progress. Must comfortably exceed +# the longest expected job (liveliness / autonomy runs). max_job_minutes: 2880 # 48 hours # --- Polling intervals (seconds) --- @@ -80,8 +91,10 @@ max_job_minutes: 2880 # 48 hours spawn_poll_interval_s: 15 reap_poll_interval_s: 30 -# How long to wait for a freshly-created server to reach ACTIVE before -# treating the spawn as failed. If Nova flips the server to ERROR within this -# window the orchestrator logs the full fault (code/message/details/host/AZ) -# and deletes the server so the next iteration can retry cleanly. -server_active_timeout_s: 300 +# How long to wait for `osmo workflow submit` to return before treating the +# submission as failed. +submit_timeout_s: 180 + +# Name prefix for submitted workflows. Also used by the orphan sweep to find +# workflows this orchestrator owns. Keep the trailing dash. +workflow_name_prefix: "gha-runner-" diff --git a/.github/orchestrator/orchestrator.py b/.github/orchestrator/orchestrator.py index 3e65e906f..4af62555b 100644 --- a/.github/orchestrator/orchestrator.py +++ b/.github/orchestrator/orchestrator.py @@ -1,56 +1,64 @@ #!/usr/bin/env python3 -"""AirStack CI orchestrator. +"""AirStack CI orchestrator (OSMO backend). Polls the GitHub API for queued workflow_jobs whose labels match this -orchestrator's runner_labels, and spawns truly ephemeral OpenStack instances -to execute them. Each ephemeral instance receives a single-use GitHub JIT -runner config via cloud-init; the GitHub PAT never leaves this orchestrator. +orchestrator's runner_labels, and submits truly ephemeral OSMO workflows to +execute them. Each workflow runs a single-job GitHub Actions runner in a +privileged, GPU-enabled container on an OSMO compute pool; the GitHub PAT never +leaves this orchestrator, and an OSMO service-account token (not a personal +account) is used only to submit / query / cancel workflows. + +This is a drop-in replacement for the previous OpenStack-Nova backend: the +GitHub side is unchanged (`runs-on: [self-hosted, airstack-ephemeral]`, the +single-use JIT runner config, the same-repo fork guard). Only the *spawn* +target changed from "create a Nova VM" to "submit an OSMO workflow". The +one-job-per-worker, destroy-after semantics are preserved — when the runner's +`run.sh` exits after a single job, the OSMO task completes and the pod is torn +down. Two cooperating loops: - - spawn loop: discover queued jobs, spawn one Nova server per job - - reap loop: delete servers whose jobs have completed, plus stragglers + - spawn loop: discover queued jobs, submit one OSMO workflow per job + - reap loop: cancel workflows whose jobs have completed, plus stragglers older than max_job_minutes and orphans not in state.json State persists in /var/lib/airstack-orchestrator/state.json so the -orchestrator can survive restarts without leaking instances. +orchestrator can survive restarts without leaking workflows. """ from __future__ import annotations import argparse -import base64 import json import logging import os +import re import signal +import subprocess import sys +import tempfile import threading import time from datetime import datetime, timezone from pathlib import Path from typing import Any -import openstack import requests import yaml from jinja2 import Template DEFAULT_CONFIG_PATH = "/etc/airstack-orchestrator/config.yaml" DEFAULT_PAT_PATH = "/etc/airstack-orchestrator/github-pat" +DEFAULT_OSMO_TOKEN_PATH = "/etc/airstack-orchestrator/osmo-token" DEFAULT_STATE_PATH = "/var/lib/airstack-orchestrator/state.json" -DEFAULT_TEMPLATE_PATH = "/opt/airstack-orchestrator/cloud-init.yaml.j2" - -# Metadata key/value applied to every Nova server we spawn. Used by the -# orphan reaper to identify servers we own even when state.json is missing. -ROLE_META_KEY = "airstack-role" -ROLE_META_VAL = "ephemeral-runner" -JOB_META_KEY = "airstack-job-id" +DEFAULT_TEMPLATE_PATH = "/opt/airstack-orchestrator/runner-workflow.yaml.j2" GITHUB_API = "https://api.github.com" log = logging.getLogger("orchestrator") +# ── file / state helpers ──────────────────────────────────────────────────── + def load_yaml(path: str) -> dict: with open(path) as f: return yaml.safe_load(f) @@ -76,6 +84,16 @@ def save_state(path: str, state: dict) -> None: os.replace(tmp, path) +def now_utc_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def parse_iso(s: str) -> datetime: + return datetime.fromisoformat(s) + + +# ── GitHub API (unchanged from the OpenStack backend) ───────────────────────── + def gh_request(method: str, path: str, pat: str, **kwargs: Any) -> Any: url = f"{GITHUB_API}{path}" headers = kwargs.pop("headers", {}) @@ -156,267 +174,134 @@ def get_job_status(repo: str, job_id: str, pat: str) -> dict | None: return r.json() -def render_cloud_init(template_path: str, encoded_jit_config: str, - runner_version: str) -> str: - with open(template_path) as f: - tmpl = Template(f.read()) - return tmpl.render( - encoded_jit_config=encoded_jit_config, - runner_version=runner_version, - ) - - -def spawn_server( - conn: openstack.connection.Connection, - config: dict, - name: str, - job_id: str, - user_data: str, -) -> str: - flavor = conn.compute.find_flavor(config["flavor_name"], ignore_missing=False) - network = conn.network.find_network(config["network_name"], ignore_missing=False) - create_kwargs = dict( - name=name, - flavor_id=flavor.id, - networks=[{"uuid": network.id}], - key_name=config["keypair_name"], - security_groups=[{"name": config["security_group"]}], - user_data=base64.b64encode(user_data.encode()).decode(), - metadata={ - ROLE_META_KEY: ROLE_META_VAL, - JOB_META_KEY: job_id, - }, - ) - - # Flavors with disk=0 (typical for GPU flavors on this cloud) cannot boot - # directly from an image — Nova rejects with "Block Device Mapping is - # Invalid: You specified more local devices than the limit allows". When - # boot_volume_size_gb is set, boot from a Cinder volume sourced from the - # image and delete it on termination. Otherwise fall back to direct image - # boot (works only when the flavor has a non-zero root disk). - boot_volume_size_gb = int(config.get("boot_volume_size_gb") or 0) - if boot_volume_size_gb > 0: - create_kwargs["block_device_mapping"] = [ - { - "uuid": config["image_id"], - "source_type": "image", - "destination_type": "volume", - "boot_index": 0, - "volume_size": boot_volume_size_gb, - "delete_on_termination": True, - } - ] - else: - create_kwargs["image_id"] = config["image_id"] - - az = config.get("availability_zone") - if az: - create_kwargs["availability_zone"] = az - server = conn.compute.create_server(**create_kwargs) - return server.id - - -def delete_server(conn: openstack.connection.Connection, server_id: str) -> None: +# ── OSMO CLI output parsing ─────────────────────────────────────────────────── +# +# The exact JSON keys returned by `osmo workflow {submit,query,list}` can vary +# slightly by OSMO release, so these parsers try a set of likely keys and fall +# back to scraping the human-readable text output. Verify the keys against your +# deployed version with `osmo workflow submit --dry-run` / `--format-type json` +# once and simplify if desired. + +# Live AirLab OSMO 6.2.x returns workflow_uuid on list/query; submit may use +# workflow_id / id / name. Prefer uuid-like keys before "name" so we don't +# accidentally treat the human workflow name as the id when both are present. +_WF_ID_KEYS = ( + "workflow_uuid", "workflow_id", "workflowId", "id", "uuid", "name", "workflow", +) +_STATUS_KEYS = ("status", "state", "workflow_status", "phase") +_KNOWN_STATUSES = { + "RUNNING", "PENDING", "WAITING", "COMPLETED", "FAILED", + "FAILED_EXEC_TIMEOUT", "FAILED_SERVER_ERROR", "FAILED_QUEUE_TIMEOUT", + "FAILED_SUBMISSION", "FAILED_CANCELED", "FAILED_BACKEND_ERROR", + "FAILED_IMAGE_PULL", "FAILED_EVICTED", "FAILED_START_ERROR", + "FAILED_START_TIMEOUT", "FAILED_PREEMPTED", +} +# Non-terminal statuses the orphan sweep considers "still alive". +_ACTIVE_STATUSES = ("RUNNING", "PENDING", "WAITING") + + +def _loads_or_none(text: str | None) -> Any: try: - conn.compute.delete_server(server_id, ignore_missing=True, force=True) - except Exception as e: - log.warning("delete_server(%s) failed: %s", server_id, e) - - -def list_owned_servers(conn: openstack.connection.Connection) -> list[Any]: - """List all Nova servers that carry our role metadata.""" - owned = [] - for s in conn.compute.servers(details=True): - meta = getattr(s, "metadata", None) or {} - if meta.get(ROLE_META_KEY) == ROLE_META_VAL: - owned.append(s) - return owned - - -def find_free_floating_ip( - conn: openstack.connection.Connection, pool: list[str] -) -> Any: - """Return the FloatingIP resource for the first address in `pool` that is - not currently associated with any port. Returns None if all are in use. - - Iterates `pool` in order so attachments rotate through it sequentially. - Logs a warning for any pool member that doesn't exist in this project. - """ - if not pool: + return json.loads(text) # type: ignore[arg-type] + except (json.JSONDecodeError, TypeError): return None - pool_set = set(pool) - fips_by_addr: dict[str, Any] = {} - for fip in conn.network.ips(): - if fip.floating_ip_address in pool_set: - fips_by_addr[fip.floating_ip_address] = fip - missing = pool_set - fips_by_addr.keys() - if missing: - log.warning( - "floating_ips configured but not found in this project: %s", - sorted(missing), - ) - for addr in pool: - fip = fips_by_addr.get(addr) - if fip is not None and not fip.port_id: - return fip - return None - -def check_flavor_capacity( - conn: openstack.connection.Connection, - flavor_name: str, -) -> tuple[bool, str]: - """Pre-flight: ask Nova's placement API whether any host can satisfy this - flavor's resource request right now. - Returns (ok, reason). When ok=False the orchestrator should defer the - spawn iteration; reason is a one-line human-readable explanation - (e.g. "no host can satisfy {'VCPU': 8, 'MEMORY_MB': 32768, 'VGPU': 1}"). - - If the placement API can't be queried for any reason we return - (True, "") and let Nova make the call. The - pre-flight is a fast-path optimization, not a gate — Nova still has the - final say at create_server time (and ERROR-status fallback handles - anything we miss). - """ - try: - flavor = conn.compute.find_flavor(flavor_name, ignore_missing=False) - except Exception as e: - return True, f"flavor lookup failed: {e}" - - # Standard resources every Nova flavor expresses. - resources: dict[str, int] = {} - if getattr(flavor, "vcpus", 0): - resources["VCPU"] = int(flavor.vcpus) - if getattr(flavor, "ram", 0): - resources["MEMORY_MB"] = int(flavor.ram) - if getattr(flavor, "disk", 0): - resources["DISK_GB"] = int(flavor.disk) - - # Custom / specialized resources (VGPU, PCI_*, CUSTOM_*) come from the - # flavor's extra_specs as `resources:=`. This is how Nova - # itself learns to ask placement for GPU capacity. - extra = getattr(flavor, "extra_specs", {}) or {} - for k, v in extra.items(): - if not k.startswith("resources:"): - continue - rc = k.split(":", 1)[1] - try: - resources[rc] = int(v) - except (TypeError, ValueError): - pass - - if not resources: - return True, "flavor expresses no resources — skipping placement check" +def _first_str(d: dict, keys: tuple[str, ...]) -> str | None: + for k in keys: + v = d.get(k) + if isinstance(v, str) and v: + return v + return None - try: - result = conn.placement.allocation_candidates( - resources=resources, limit=1, - ) - if hasattr(result, "allocation_requests"): - candidates = list(result.allocation_requests or []) - else: - candidates = list(result) - except Exception as e: - return True, f"placement query failed ({type(e).__name__}: {e})" - - if candidates: - return True, "" - return False, f"no host can satisfy {resources}" - - -def wait_for_server_active( - conn: openstack.connection.Connection, - server_id: str, - timeout_s: int = 300, - poll_interval_s: float = 3.0, -) -> Any: - """Poll Nova until the server is ACTIVE. Raise with full context if it - enters ERROR or never reaches ACTIVE in time. - - Nova surfaces the actual reason for an ERROR via the `fault` attribute - (message + code + details), so we log it verbatim. We also include - task_state / vm_state / power_state because Nova sometimes leaves the - fault empty and these tell you whether the failure was at scheduling, - networking, or block-device-mapping time. - """ - deadline = time.monotonic() + timeout_s - last_status = "?" - last_task = None - while time.monotonic() < deadline: - s = conn.compute.get_server(server_id) - status = getattr(s, "status", "UNKNOWN") or "UNKNOWN" - task = ( - getattr(s, "task_state", None) - or getattr(s, "OS-EXT-STS:task_state", None) - ) - if status != last_status or task != last_task: - log.info( - "server %s status=%s task_state=%s", server_id, status, task, - ) - last_status, last_task = status, task - if status == "ACTIVE": +def _extract_workflow_id(stdout: str | None) -> str | None: + data = _loads_or_none(stdout) + if isinstance(data, dict): + wid = _first_str(data, _WF_ID_KEYS) + if wid: + return wid + wf = data.get("workflow") + if isinstance(wf, dict): + wid = _first_str(wf, _WF_ID_KEYS) + if wid: + return wid + m = re.search(r"Workflow\s*ID\s*[-:]\s*(\S+)", stdout or "", re.IGNORECASE) + return m.group(1) if m else None + + +def _extract_status(stdout: str | None) -> str | None: + data = _loads_or_none(stdout) + if isinstance(data, dict): + st = _first_str(data, _STATUS_KEYS) + if st: + return st.upper() + wf = data.get("workflow") + if isinstance(wf, dict): + st = _first_str(wf, _STATUS_KEYS) + if st: + return st.upper() + up = (stdout or "").upper() + for s in sorted(_KNOWN_STATUSES, key=len, reverse=True): + if s in up: return s + return None - if status == "ERROR": - fault = getattr(s, "fault", None) or {} - vm_state = ( - getattr(s, "vm_state", None) - or getattr(s, "OS-EXT-STS:vm_state", None) - ) - power_state = ( - getattr(s, "power_state", None) - or getattr(s, "OS-EXT-STS:power_state", None) - ) - host = getattr(s, "compute_host", None) or getattr( - s, "OS-EXT-SRV-ATTR:host", None - ) - az = getattr(s, "availability_zone", None) or getattr( - s, "OS-EXT-AZ:availability_zone", None - ) - raise RuntimeError( - "server " - + str(server_id) - + " entered ERROR: " - + f"fault.code={fault.get('code')!r} " - + f"fault.message={fault.get('message')!r} " - + f"fault.details={fault.get('details')!r} " - + f"task_state={task!r} vm_state={vm_state!r} " - + f"power_state={power_state!r} host={host!r} az={az!r}" - ) - time.sleep(poll_interval_s) +def _extract_workflow_list(stdout: str | None) -> list[dict]: + data = _loads_or_none(stdout) + if isinstance(data, dict): + for key in ("workflows", "items", "results", "data"): + if isinstance(data.get(key), list): + data = data[key] + break + items: list[dict] = [] + if isinstance(data, list): + for entry in data: + if not isinstance(entry, dict): + continue + wid = _first_str(entry, _WF_ID_KEYS) + name = entry.get("name") if isinstance(entry.get("name"), str) else None + status = _first_str(entry, _STATUS_KEYS) + if wid or name: + items.append( + {"id": wid, "name": name, + "status": status.upper() if status else None} + ) + return items + - raise RuntimeError( - f"server {server_id} did not reach ACTIVE within {timeout_s}s " - f"(last status={last_status!r} task_state={last_task!r})" - ) +def _is_terminal(status: str | None) -> bool: + if not status: + return False + return status == "COMPLETED" or status.startswith("FAILED") -def attach_floating_ip( - conn: openstack.connection.Connection, server_id: str, fip: Any -) -> str: - """Wait for the server to have a network port, then associate `fip`. - Returns the floating IP address.""" - for _ in range(60): # ~120s - ports = list(conn.network.ports(device_id=server_id)) - if ports: - break - time.sleep(2) - else: - raise RuntimeError(f"server {server_id} got no network port within 120s") - conn.network.update_ip(fip, port_id=ports[0].id) - return fip.floating_ip_address +def _looks_like_auth_error(r: subprocess.CompletedProcess) -> bool: + blob = f"{r.stdout or ''}\n{r.stderr or ''}".lower() + markers = ("401", "403", "unauthorized", "forbidden", "expired", + "not logged in", "please login", "authentication", + "invalid token", "token is invalid") + return any(m in blob for m in markers) -def now_utc_iso() -> str: - return datetime.now(timezone.utc).isoformat() +def _name_age_minutes(name: str | None) -> float | None: + """Age in minutes parsed from our `...-` name suffix, or None. + OSMO may append its own suffix after the name we submit, so we match the + first 10+ digit run (the unix timestamp) even when trailing chars follow. + """ + m = re.search(r"-(\d{10,})(?:\D.*)?$", name or "") + if not m: + return None + try: + ts = int(m.group(1)) + except ValueError: + return None + return (time.time() - ts) / 60.0 -def parse_iso(s: str) -> datetime: - return datetime.fromisoformat(s) +# ── orchestrator ────────────────────────────────────────────────────────────── class Orchestrator: def __init__(self, config: dict, pat: str, state_path: str, template_path: str): @@ -424,230 +309,308 @@ def __init__(self, config: dict, pat: str, state_path: str, template_path: str): self.pat = pat self.state_path = state_path self.template_path = template_path - self.conn = openstack.connect(cloud=config.get("openstack_cloud", "airstack")) + + # OSMO target. + self.osmo_bin = config.get("osmo_bin", "osmo") + self.osmo_url = config["osmo_url"] + self.token_file = config.get("osmo_token_file", DEFAULT_OSMO_TOKEN_PATH) + self.pool = config["pool"] + self.platform = config.get("platform", "") or "" + self.priority = str(config.get("priority", "NORMAL")).upper() + + # Runner task shape. + self.runner_image = config["runner_image"] + self.cpu = config.get("cpu", 8) + self.gpu = config.get("gpu", 1) + self.memory = config.get("memory", "32Gi") + self.storage = config.get("storage", "300Gi") + self.privileged = bool(config.get("privileged", True)) + self.host_network = bool(config.get("host_network", False)) + + # GitHub. self.repo = config["repo"] self.runner_labels = config["runner_labels"] - self.runner_version = config["runner_version"] + + # Limits / timing. self.max_concurrent = int(config.get("max_concurrent", 3)) - self.floating_ips: list[str] = list(config.get("floating_ips") or []) - # Cap spawns to FIP pool size so we never queue jobs we can't address. - self.effective_max_concurrent = self.max_concurrent - if self.floating_ips: - self.effective_max_concurrent = min( - self.max_concurrent, len(self.floating_ips) - ) - self.max_job_minutes = int(config.get("max_job_minutes", 90)) + self.max_job_minutes = int(config.get("max_job_minutes", 2880)) self.spawn_interval = int(config.get("spawn_poll_interval_s", 15)) self.reap_interval = int(config.get("reap_poll_interval_s", 30)) + self.submit_timeout = int(config.get("submit_timeout_s", 180)) + self.workflow_prefix = config.get("workflow_name_prefix", "gha-runner-") + self.stop_evt = threading.Event() + # Establish the OSMO session up-front for early feedback; individual + # commands re-login on demand if the session lapses. + self._login() + def stop(self, *_: Any) -> None: log.info("stop signal received; draining loops") self.stop_evt.set() + # ── OSMO CLI plumbing ──────────────────────────────────────────────────── + + def _run_osmo(self, args: list[str], timeout: int) -> subprocess.CompletedProcess: + return subprocess.run( + [self.osmo_bin, *args], + capture_output=True, text=True, timeout=timeout, + ) + + def _login(self) -> bool: + try: + r = self._run_osmo( + ["login", self.osmo_url, "--method", "token", + "--token-file", self.token_file], + timeout=60, + ) + except Exception as e: # noqa: BLE001 - startup best-effort + log.warning("osmo login raised: %s", e) + return False + if r.returncode != 0: + log.warning( + "osmo login failed (rc=%d): %s", + r.returncode, (r.stderr or r.stdout).strip(), + ) + return False + log.info("osmo login succeeded (url=%s, token_file=%s)", + self.osmo_url, self.token_file) + return True + + def _osmo(self, args: list[str], timeout: int, + relogin: bool = True) -> subprocess.CompletedProcess: + """Run an osmo CLI command, re-logging-in once on an auth failure.""" + r = self._run_osmo(args, timeout=timeout) + if r.returncode != 0 and relogin and _looks_like_auth_error(r): + log.info("osmo command hit an auth error; re-logging in and retrying") + if self._login(): + r = self._run_osmo(args, timeout=timeout) + return r + + def submit_workflow(self, workflow_file: str) -> tuple[str, str]: + """Submit a workflow. Returns (workflow_id, live_name). + + AirLab OSMO 6.2 submit JSON is typically only {name, overview, logs} + (no uuid), and the service may append a numeric suffix to the name + (e.g. ``...-1``). We immediately query to resolve uuid + live name so + state/reap stay consistent with ``workflow list`` (``workflow_uuid``). + """ + args = ["workflow", "submit", workflow_file, "--pool", self.pool, + "--priority", self.priority, "--format-type", "json"] + r = self._osmo(args, timeout=self.submit_timeout) + if r.returncode != 0: + raise RuntimeError( + f"osmo workflow submit failed (rc={r.returncode}): " + f"{(r.stderr or r.stdout).strip()}" + ) + submitted_name = _extract_workflow_id(r.stdout) or _extract_workflow_id(r.stderr) + if not submitted_name: + raise RuntimeError( + "could not parse workflow id/name from submit output: " + f"{(r.stdout or '').strip()[:500]}" + ) + live_name, uuid = submitted_name, None + q = self._osmo( + ["workflow", "query", submitted_name, "--format-type", "json"], + timeout=60, + ) + if q.returncode == 0: + data = _loads_or_none(q.stdout) or _loads_or_none(q.stderr) + if isinstance(data, dict): + if isinstance(data.get("name"), str) and data["name"]: + live_name = data["name"] + for k in ("uuid", "workflow_uuid", "workflow_id", "id"): + v = data.get(k) + if isinstance(v, str) and v: + uuid = v + break + return (uuid or live_name), live_name + + def query_status(self, workflow_id: str) -> str | None: + r = self._osmo( + ["workflow", "query", workflow_id, "--format-type", "json"], + timeout=60, + ) + if r.returncode != 0: + log.debug("osmo workflow query %s failed: %s", + workflow_id, (r.stderr or r.stdout).strip()) + return None + return _extract_status(r.stdout) or _extract_status(r.stderr) + + def cancel_workflow(self, workflow_id: str) -> None: + r = self._osmo( + ["workflow", "cancel", workflow_id, "--force", + "--message", "orchestrator reap", "--format-type", "json"], + timeout=60, + ) + if r.returncode != 0: + log.warning("osmo workflow cancel %s failed (rc=%d): %s", + workflow_id, r.returncode, (r.stderr or r.stdout).strip()) + + def list_runner_workflows(self) -> list[dict]: + """Active workflows (RUNNING/PENDING/WAITING) named with our prefix.""" + r = self._osmo( + ["workflow", "list", "--name", self.workflow_prefix, + "--pool", self.pool, "--count", "100", + "--status", *_ACTIVE_STATUSES, "--format-type", "json"], + timeout=60, + ) + if r.returncode != 0: + log.warning("osmo workflow list failed: %s", + (r.stderr or r.stdout).strip()) + return [] + return _extract_workflow_list(r.stdout) + + # ── workflow rendering ─────────────────────────────────────────────────── + + def render_workflow(self, workflow_name: str, encoded_jit_config: str) -> str: + with open(self.template_path) as f: + tmpl = Template(f.read()) + return tmpl.render( + workflow_name=workflow_name, + runner_image=self.runner_image, + cpu=self.cpu, + gpu=self.gpu, + memory=self.memory, + storage=self.storage, + platform=self.platform, + privileged="true" if self.privileged else "false", + host_network="true" if self.host_network else "false", + encoded_jit_config=encoded_jit_config, + runner_labels=self.runner_labels, + ) + + def _write_temp_workflow(self, name: str, content: str) -> str: + fd, path = tempfile.mkstemp(prefix=f"{name}-", suffix=".yaml") + with os.fdopen(fd, "w") as f: + f.write(content) + return path + + # ── loops ──────────────────────────────────────────────────────────────── + def spawn_once(self) -> None: state = load_state(self.state_path) active = len(state["jobs"]) - if active >= self.effective_max_concurrent: + if active >= self.max_concurrent: return try: queued = find_queued_jobs(self.repo, self.runner_labels, self.pat) - except Exception as e: + except Exception as e: # noqa: BLE001 log.warning("find_queued_jobs failed: %s", e) return - # Pre-flight capacity check via placement API. Every queued job uses - # the same flavor, so we check once per iteration. When OpenStack is - # out of GPUs / vCPU / RAM we defer the whole iteration — better than - # burning JIT tokens on creates that Nova will flip to ERROR. The - # next iteration retries automatically. - if queued: - ok, reason = check_flavor_capacity( - self.conn, self.config["flavor_name"] - ) - if not ok: - log.warning( - "deferring spawn — OpenStack capacity unavailable: %s. " - "Will retry in %ds.", - reason, self.spawn_interval, - ) - return - elif reason: - # Soft-skip path: placement check couldn't run (e.g. older - # Nova). Surface why so it's debuggable, then proceed. - log.debug("placement pre-flight: %s", reason) - for job in queued: - if active >= self.effective_max_concurrent: + if active >= self.max_concurrent: break job_id = job["job_id"] if job_id in state["jobs"]: continue - # Pre-check FIP availability before minting a JIT token so we - # don't burn one when there's nowhere to attach the worker. - reserved_fip = None - if self.floating_ips: - reserved_fip = find_free_floating_ip(self.conn, self.floating_ips) - if reserved_fip is None: - log.warning( - "no free floating IP in pool (%d configured); " - "deferring spawns until one frees up", - len(self.floating_ips), - ) - break - ts = int(time.time()) - runner_name = f"ephemeral-{job_id}-{ts}" - server_id: str | None = None + # OSMO workflow name doubles as the JIT runner registration name. + workflow_name = f"{self.workflow_prefix}{job_id}-{ts}" + tmp_path: str | None = None try: jit = mint_jit_config( - self.repo, runner_name, self.runner_labels, self.pat + self.repo, workflow_name, self.runner_labels, self.pat ) - user_data = render_cloud_init( - self.template_path, jit, self.runner_version - ) - server_id = spawn_server( - self.conn, self.config, runner_name, job_id, user_data - ) - # Don't move on until Nova reports ACTIVE. If it transitions - # to ERROR, this raises with the Nova fault details so the - # operator can see *why* the spawn failed (quota, scheduling, - # block-device-mapping, networking, etc.). - wait_for_server_active( - self.conn, - server_id, - timeout_s=int(self.config.get("server_active_timeout_s", 300)), - ) - except Exception as e: - # Tag capacity-related Nova faults so log-grepping for - # "capacity unavailable" finds both the pre-flight defer and - # the post-create fallback (e.g. PCI passthrough that - # placement doesn't track). - msg = str(e).lower() - capacity_markers = ( - "no valid host", - "insufficient", - "quotaexceeded", - "out of resource", - "no host can satisfy", - "no allocation candidates", - ) - if any(m in msg for m in capacity_markers): - log.warning( - "spawn failed for job %s — OpenStack capacity " - "unavailable (post-create): %s. Will retry in %ds.", - job_id, e, self.spawn_interval, - ) - else: - log.exception("spawn failed for job %s: %s", job_id, e) - if server_id: - log.warning( - "deleting failed server %s to release its volume / FIP", - server_id, - ) - delete_server(self.conn, server_id) + workflow_yaml = self.render_workflow(workflow_name, jit) + tmp_path = self._write_temp_workflow(workflow_name, workflow_yaml) + workflow_id, live_name = self.submit_workflow(tmp_path) + except Exception as e: # noqa: BLE001 + log.exception("submit failed for job %s: %s", job_id, e) continue - - floating_ip_addr: str | None = None - if reserved_fip is not None: - try: - floating_ip_addr = attach_floating_ip( - self.conn, server_id, reserved_fip - ) - log.info( - "attached floating IP %s to server %s (job %s)", - floating_ip_addr, server_id, job_id, - ) - except Exception as e: - log.exception( - "FIP attach failed for server %s; deleting to avoid " - "leaking a worker without external access: %s", - server_id, e, - ) - delete_server(self.conn, server_id) - continue + finally: + if tmp_path: + try: + os.remove(tmp_path) + except OSError: + pass state["jobs"][job_id] = { "run_id": job["run_id"], - "server_id": server_id, - "runner_name": runner_name, - "spawned_at": now_utc_iso(), + "workflow_id": workflow_id, + "workflow_name": live_name, + "runner_name": workflow_name, + "submitted_at": now_utc_iso(), "name": job["name"], - "floating_ip": floating_ip_addr, } save_state(self.state_path, state) active += 1 log.info( - "spawned server %s for job %s (%s)", server_id, job_id, job["name"] + "submitted workflow %s for job %s (%s)", + workflow_id, job_id, job["name"], ) def reap_once(self) -> None: state = load_state(self.state_path) now = datetime.now(timezone.utc) - # 1. Delete servers for completed jobs. + # 1. Cancel workflows for completed / purged jobs. for job_id in list(state["jobs"].keys()): entry = state["jobs"][job_id] + wid = entry["workflow_id"] try: job = get_job_status(self.repo, job_id, self.pat) - except Exception as e: + except Exception as e: # noqa: BLE001 log.warning("get_job_status(%s) failed: %s", job_id, e) continue + if job is None or job.get("status") == "completed": - log.info("reaping server %s (job %s done)", entry["server_id"], job_id) - delete_server(self.conn, entry["server_id"]) + # The runner usually exits on its own (task self-completes and + # the pod is torn down); only cancel if it's somehow still live. + status = self.query_status(wid) + if not _is_terminal(status): + log.info("reaping workflow %s (job %s done, wf status=%s)", + wid, job_id, status) + self.cancel_workflow(wid) + else: + log.info("workflow %s already terminal (%s) for job %s", + wid, status, job_id) del state["jobs"][job_id] continue # 2. Force-reap stragglers older than max_job_minutes. - spawned = parse_iso(entry["spawned_at"]) - age_min = (now - spawned).total_seconds() / 60.0 + age_min = (now - parse_iso(entry["submitted_at"])).total_seconds() / 60.0 if age_min > self.max_job_minutes: log.warning( - "force-reaping server %s (job %s age %.1fm > %dm)", - entry["server_id"], job_id, age_min, self.max_job_minutes, + "force-reaping workflow %s (job %s age %.1fm > %dm)", + wid, job_id, age_min, self.max_job_minutes, ) - delete_server(self.conn, entry["server_id"]) + self.cancel_workflow(wid) del state["jobs"][job_id] save_state(self.state_path, state) - # 3. Orphan sweep: any server we own that isn't in state and isn't - # in the brief just-spawned window. Catches state.json wipes and - # crashes between spawn and save_state. + # 3. Orphan sweep: our-named workflows still active but absent from + # state (catches state.json wipes and crashes between submit and + # save_state). Skip very fresh ones so we don't race our own submit. try: - owned = list_owned_servers(self.conn) - except Exception as e: - log.warning("list_owned_servers failed: %s", e) + listed = self.list_runner_workflows() + except Exception as e: # noqa: BLE001 + log.warning("list_runner_workflows failed: %s", e) return - tracked_ids = {e["server_id"] for e in state["jobs"].values()} - for s in owned: - if s.id in tracked_ids: + tracked_ids = {e["workflow_id"] for e in state["jobs"].values()} + tracked_names = {e["workflow_name"] for e in state["jobs"].values()} + for wf in listed: + wid, wname = wf.get("id"), wf.get("name") + if (wid and wid in tracked_ids) or (wname and wname in tracked_names): continue - created = getattr(s, "created_at", None) - if created: - try: - age_min = (now - parse_iso(created.replace("Z", "+00:00"))).total_seconds() / 60.0 - except Exception: - age_min = self.max_job_minutes + 1 - else: - age_min = self.max_job_minutes + 1 - # Only reap orphans that have lived past one spawn interval - # (to avoid racing our own freshly-created server). - if age_min < 2: + age = _name_age_minutes(wname) + if age is not None and age < 2: continue - log.warning( - "orphan-reaping server %s (not in state, age %.1fm)", s.id, age_min - ) - delete_server(self.conn, s.id) + target = wid or wname + if not target: + continue + log.warning("orphan-reaping workflow %s (not in state)", target) + self.cancel_workflow(target) def run(self) -> None: log.info( - "orchestrator started: repo=%s labels=%s max_concurrent=%d " - "(effective=%d, floating_ip_pool=%d)", - self.repo, self.runner_labels, self.max_concurrent, - self.effective_max_concurrent, len(self.floating_ips), + "orchestrator started (OSMO backend): repo=%s labels=%s pool=%s " + "platform=%s max_concurrent=%d", + self.repo, self.runner_labels, self.pool, + self.platform or "(pool default)", self.max_concurrent, ) last_spawn = 0.0 last_reap = 0.0 @@ -656,13 +619,13 @@ def run(self) -> None: if now - last_spawn >= self.spawn_interval: try: self.spawn_once() - except Exception: + except Exception: # noqa: BLE001 log.exception("spawn loop iteration failed") last_spawn = now if now - last_reap >= self.reap_interval: try: self.reap_once() - except Exception: + except Exception: # noqa: BLE001 log.exception("reap loop iteration failed") last_reap = now self.stop_evt.wait(timeout=1.0) diff --git a/.github/orchestrator/requirements.txt b/.github/orchestrator/requirements.txt index b69702b59..f710f7167 100644 --- a/.github/orchestrator/requirements.txt +++ b/.github/orchestrator/requirements.txt @@ -1,4 +1,3 @@ -openstacksdk>=3.0,<5 requests>=2.31 PyYAML>=6.0 Jinja2>=3.1 diff --git a/.github/orchestrator/runner-entrypoint.sh b/.github/orchestrator/runner-entrypoint.sh new file mode 100644 index 000000000..3026dbdf4 --- /dev/null +++ b/.github/orchestrator/runner-entrypoint.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Entry point for the AirStack CI ephemeral-runner container (an OSMO task). +# +# Starts an inner Docker daemon — the AirStack test harness runs `airstack up` +# (docker compose) inside this container — waits for it, then runs exactly ONE +# ephemeral GitHub Actions job via the single-use JIT config. When run.sh exits, +# the OSMO task completes and the pod is destroyed: one job per pod, same as the +# old OpenStack VM. +# +# Requires a privileged pod (dockerd) scheduled on a GPU platform; the NVIDIA +# container toolkit (baked into the image) lets the inner dockerd pass the node +# GPU through to the compose containers. +set -euo pipefail + +: "${ENCODED_JIT_CONFIG:?ENCODED_JIT_CONFIG must be set by the workflow}" + +log() { echo "[runner-entrypoint] $*"; } + +# --------------------------------------------------------------------------- +# Docker storage backend +# +# The pod's root filesystem is overlayfs, and Linux refuses to use a directory +# on overlayfs as an overlay `upperdir` (EINVAL). A dockerd whose data-root sits +# on the pod rootfs still *pulls* images fine — containerd unpacks layers with +# plain writes — but every build step that needs a real mount dies with: +# +# failed to solve: ... mount source: "overlay", +# target: "/var/lib/docker/buildkit/containerd-overlayfs/cachemounts/...", +# err: invalid argument +# +# which is what took out all of build_docker / build_packages. So put the +# data-root somewhere overlay actually works, and verify it by performing a real +# overlay mount rather than trusting the filesystem type. +# --------------------------------------------------------------------------- + +DOCKER_DATA_ROOT=/var/lib/docker +LOOP_IMG=/docker-data.img +# Headroom left for everything that is not the Docker data-root (the runner's +# _work checkout, logs, the loop image's own metadata). +LOOP_HEADROOM_MB=20480 +MIN_BACKING_MB=51200 # Below ~50 GiB the sim images can't fit regardless. + +# True if an overlay mount whose upperdir lives under $1 can actually be made. +overlay_upperdir_works() { + local base=$1 probe rc=1 + probe=$(mktemp -d "$base/.overlay-probe.XXXXXX" 2>/dev/null) || return 1 + mkdir -p "$probe"/{lower,upper,work,merged} + if mount -t overlay overlay \ + -o "lowerdir=$probe/lower,upperdir=$probe/upper,workdir=$probe/work" \ + "$probe/merged" 2>/dev/null; then + umount "$probe/merged" && rc=0 + fi + rm -rf "$probe" + return $rc +} + +free_mb() { df -Pm "$1" | awk 'NR==2 {print $4}'; } + +# Preferred: a loopback ext4 image mounted at the data-root. Self-contained +# (no dependency on how the pool exposes storage), gives real overlay2, and dies +# with the pod. The image file is sparse, so it only consumes what Docker writes. +setup_loopback() { + local size_mb=${DOCKER_LOOP_SIZE_MB:-} + if [[ -z "$size_mb" ]]; then + size_mb=$(( $(free_mb /) - LOOP_HEADROOM_MB )) + fi + if (( size_mb < MIN_BACKING_MB )); then + log "loopback: only ${size_mb}MB usable, need ${MIN_BACKING_MB}MB — skipping" + return 1 + fi + + # /dev/loop-control only exists once the loop module is loaded on the node. + [[ -e /dev/loop-control ]] || modprobe loop 2>/dev/null || true + if [[ ! -e /dev/loop-control ]]; then + log "loopback: no /dev/loop-control — skipping" + return 1 + fi + + log "loopback: creating ${size_mb}MB ext4 image at $LOOP_IMG" + truncate -s "${size_mb}M" "$LOOP_IMG" || return 1 + # No journal and lazy inode-table init: this filesystem never outlives the + # pod, so durability buys nothing and mkfs stays fast. + mkfs.ext4 -q -F -m 0 -O ^has_journal -E lazy_itable_init=1 "$LOOP_IMG" || return 1 + + mkdir -p "$DOCKER_DATA_ROOT" + mount -o loop "$LOOP_IMG" "$DOCKER_DATA_ROOT" || return 1 + + if ! overlay_upperdir_works "$DOCKER_DATA_ROOT"; then + log "loopback: mounted but overlay still rejected — unwinding" + umount "$DOCKER_DATA_ROOT" || true + rm -f "$LOOP_IMG" + return 1 + fi + return 0 +} + +# Fallback: a real filesystem already mounted into the pod. Kubernetes emptyDir, +# hostPath and PVC volumes are backed by the node disk rather than the overlay +# rootfs, so overlay works there. +setup_real_fs() { + local best="" best_free=0 mnt fstype opts avail + while read -r _ mnt fstype opts _; do + case "$fstype" in ext2|ext3|ext4|xfs|btrfs) ;; *) continue ;; esac + [[ -d "$mnt" && -w "$mnt" ]] || continue + [[ ",$opts," == *",ro,"* ]] && continue + # OSMO's ctrl sidecar owns these: /osmo/data/output is uploaded as job + # artifacts and the socket dir is its IPC channel. + case "$mnt" in /osmo/data/output*|/osmo/data/socket*) continue ;; esac + avail=$(free_mb "$mnt") + if (( avail > best_free )); then best_free=$avail; best=$mnt; fi + done < /proc/mounts + + if [[ -z "$best" ]] || (( best_free < MIN_BACKING_MB )); then + log "real-fs: no mounted filesystem with >=${MIN_BACKING_MB}MB free — skipping" + return 1 + fi + + DOCKER_DATA_ROOT="$best/airstack-docker-data" + mkdir -p "$DOCKER_DATA_ROOT" + if ! overlay_upperdir_works "$DOCKER_DATA_ROOT"; then + log "real-fs: overlay rejected under $best — skipping" + DOCKER_DATA_ROOT=/var/lib/docker + return 1 + fi + log "real-fs: using $DOCKER_DATA_ROOT (${best_free}MB free)" + return 0 +} + +mkdir -p /etc/docker +if setup_loopback; then + log "storage: overlay2 on a loopback ext4 image" + printf '{"data-root": "%s"}\n' "$DOCKER_DATA_ROOT" > /etc/docker/daemon.json +elif setup_real_fs; then + log "storage: overlay2 on $DOCKER_DATA_ROOT" + printf '{"data-root": "%s"}\n' "$DOCKER_DATA_ROOT" > /etc/docker/daemon.json +elif [[ -e /dev/fuse ]] && command -v fuse-overlayfs >/dev/null 2>&1; then + # fuse-overlayfs stacks on overlayfs where the kernel driver won't. Slower + # than overlay2 but nowhere near as bad as vfs. `storage-driver` only applies + # to the classic image store, so the containerd snapshotter has to go. + log "storage: fuse-overlayfs (no overlay-capable filesystem found)" + cat > /etc/docker/daemon.json <<'JSON' +{"storage-driver": "fuse-overlayfs", "features": {"containerd-snapshotter": false}} +JSON +else + # Always works, but copies the whole filesystem per layer. The sim images are + # large enough that this will likely exhaust the pod's storage request. + log "WARN: storage: falling back to vfs — builds will be slow and may run out of disk" + cat > /etc/docker/daemon.json <<'JSON' +{"storage-driver": "vfs", "features": {"containerd-snapshotter": false}} +JSON +fi + +# Start dockerd in the background (needs privileged). +dockerd >/var/log/dockerd.log 2>&1 & + +# Wait for the daemon to accept connections (~60s budget). +for _ in $(seq 1 60); do + if docker info >/dev/null 2>&1; then + break + fi + sleep 1 +done +if ! docker info >/dev/null 2>&1; then + echo "ERROR: dockerd did not become ready" >&2 + cat /var/log/dockerd.log >&2 || true + exit 1 +fi + +log "storage driver: $(docker info --format '{{.Driver}}' 2>/dev/null || echo unknown)" \ + "data-root: $(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo unknown)" + +# Non-fatal GPU sanity check — surfaces GPU/privileged/toolkit misconfig early. +nvidia-smi || echo "WARN: nvidia-smi unavailable (check GPU + privileged + toolkit)" + +cd /home/runner/actions-runner +# The JIT config makes this runner single-use + ephemeral; run.sh returns after +# one job, which completes the task and lets OSMO reap the pod. +exec ./run.sh --jitconfig "${ENCODED_JIT_CONFIG}" diff --git a/.github/orchestrator/runner-workflow.yaml.j2 b/.github/orchestrator/runner-workflow.yaml.j2 new file mode 100644 index 000000000..124210ddb --- /dev/null +++ b/.github/orchestrator/runner-workflow.yaml.j2 @@ -0,0 +1,48 @@ +# OSMO workflow rendered per-job by orchestrator.py (replaces the old +# cloud-init.yaml.j2). One workflow == one ephemeral GitHub Actions runner == +# one CI job. Jinja variables injected by the orchestrator: +# +# workflow_name unique name (gha-runner--); also the +# JIT runner registration name +# runner_image prebaked image (docker-ce + compose + nvidia-container- +# toolkit + GH Actions runner) — see runner.Dockerfile +# cpu / gpu / memory / storage resource request for the runner task +# platform optional OSMO platform to target within the pool +# (omitted -> the pool's default platform) +# privileged "true"/"false"; MUST be "true" because the AirStack +# test harness runs `airstack up` (docker compose) inside +# the pod, which needs an inner Docker daemon. Requires a +# platform with "Privileged Mode Allowed" (ask your OSMO +# admin to enable it for the CI pool). +# host_network "true"/"false" +# encoded_jit_config single-use base64 GitHub JIT runner config +# +# The pool is passed by the orchestrator via `osmo workflow submit --pool`, so +# it is intentionally not hard-coded here. +# +# Lifecycle: the container starts dockerd, then runs exactly ONE ephemeral job +# via the JIT config. When run.sh exits, the task completes and OSMO tears the +# pod down — same "destroy after one job" behavior the OpenStack VM had. +workflow: + name: {{ workflow_name }} + resources: + runner: + cpu: {{ cpu }} + gpu: {{ gpu }} + memory: {{ memory }} + storage: {{ storage }} +{% if platform %} platform: {{ platform }} +{% endif %} + tasks: + - name: runner + image: {{ runner_image }} + resource: runner + privileged: {{ privileged }} +{% if host_network == "true" %} hostNetwork: true +{% endif %} + environment: + # Single-use + ephemeral: the runner exits after exactly one job. + ENCODED_JIT_CONFIG: "{{ encoded_jit_config }}" + # The runner refuses to run as root without this; the DinD image is root. + RUNNER_ALLOW_RUNASROOT: "1" + command: ["/usr/local/bin/run-ephemeral-runner.sh"] diff --git a/.github/orchestrator/runner.Dockerfile b/.github/orchestrator/runner.Dockerfile new file mode 100644 index 000000000..18bcf8000 --- /dev/null +++ b/.github/orchestrator/runner.Dockerfile @@ -0,0 +1,67 @@ +# Prebaked image for AirStack CI ephemeral runners on OSMO. +# +# This bakes in what the old cloud-init.yaml.j2 installed on the OpenStack VM +# (Docker CE + compose plugin, NVIDIA container toolkit, the GitHub Actions +# runner) so pod start is fast and the single-use JIT token can't expire during +# a slow apt/bootstrap. Build it and push to a registry your OSMO pool can pull: +# +# docker build -f runner.Dockerfile \ +# --build-arg RUNNER_VERSION=2.336.0 \ +# -t /airstack-ci-runner:2.336.0 . +# docker push /airstack-ci-runner:2.336.0 +# +# Then set `runner_image: /airstack-ci-runner:2.336.0` in config.yaml. +# Keep RUNNER_VERSION in sync with the actions/runner release you want. +# +# GPU-in-Docker-in-Docker: the OSMO task must run privileged (see +# `privileged: true` in runner-workflow.yaml.j2) on a platform with +# "Privileged Mode Allowed" + GPUs. The inner dockerd uses the NVIDIA container +# toolkit installed here to expose the node GPU to the `airstack up` containers. +# The image is linux/amd64 (x86_64 runner tarball); rebuild for arm64 if needed. +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# Docker CE (+ compose/buildx plugins), NVIDIA container toolkit, and the tools +# the AirStack test harness / GH runner need (git, jq, python venv, ...). +# e2fsprogs + fuse-overlayfs back the storage-backend selection in +# runner-entrypoint.sh: the pod rootfs is overlayfs, which the kernel rejects as +# an overlay upperdir, so dockerd's data-root has to live on a loopback ext4 +# image (mkfs.ext4) or, failing that, use the fuse-overlayfs driver. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl gnupg jq git sudo iproute2 \ + e2fsprogs fuse-overlayfs kmod mount \ + python3 python3-venv python3-pip \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ +https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + > /etc/apt/sources.list.d/docker.list \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ + | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ + > /etc/apt/sources.list.d/nvidia-container-toolkit.list \ + && apt-get update && apt-get install -y --no-install-recommends \ + docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin \ + nvidia-container-toolkit \ + && nvidia-ctk runtime configure --runtime=docker \ + && rm -rf /var/lib/apt/lists/* + +# GitHub Actions runner (self-contained; version pinned at build time). +ARG RUNNER_VERSION=2.336.0 +RUN mkdir -p /home/runner/actions-runner \ + && cd /home/runner/actions-runner \ + && curl -fsSL -o runner.tar.gz \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" \ + && tar xzf runner.tar.gz \ + && rm runner.tar.gz \ + && ./bin/installdependencies.sh + +COPY runner-entrypoint.sh /usr/local/bin/run-ephemeral-runner.sh +RUN chmod +x /usr/local/bin/run-ephemeral-runner.sh + +WORKDIR /home/runner/actions-runner diff --git a/.github/orchestrator/setup.sh b/.github/orchestrator/setup.sh index 4803b4a33..adb3c078f 100755 --- a/.github/orchestrator/setup.sh +++ b/.github/orchestrator/setup.sh @@ -1,13 +1,17 @@ #!/usr/bin/env bash -# One-time orchestrator-VM setup. Run as root on the airstack-ci-cd-orchestrator -# OpenStack instance after cloning the repo. +# One-time orchestrator-VM setup (OSMO backend). Run as root on the +# airstack-ci-cd-orchestrator instance after cloning the repo. # # Pre-reqs (do these *before* running this script): -# 1. ~/.config/openstack/clouds.yaml staged for the orchestrator user -# (application credential — see .github/orchestrator/README.md). -# 2. /tmp/github-pat exists with the GitHub PAT contents. +# 1. /tmp/github-pat exists with the GitHub PAT contents. +# 2. /tmp/osmo-token exists with the OSMO service-account access token +# (from `osmo token set` — see .github/orchestrator/README.md). Optional +# at setup time; you can stage it later before starting the service. # 3. This repo cloned somewhere readable (this script copies code out of # its containing directory). +# +# The orchestrator host is lightweight and needs NO GPU — it only polls GitHub +# and submits OSMO workflows. 1 vCPU / 2GB RAM / 20GB disk is plenty. set -euo pipefail @@ -30,18 +34,33 @@ fi echo "==> Installing system packages" apt-get update -apt-get install -y python3 python3-venv python3-pip +apt-get install -y python3 python3-venv python3-pip curl ca-certificates + +echo "==> Installing the OSMO CLI" +if command -v osmo >/dev/null 2>&1; then + echo " osmo already installed ($(command -v osmo)); skipping" +else + # Latest client. Pin to a release from https://github.com/NVIDIA/OSMO/releases + # if you need a specific version. + curl -fsSL https://raw.githubusercontent.com/NVIDIA/OSMO/refs/heads/main/install.sh | bash + command -v osmo >/dev/null 2>&1 \ + || echo "WARNING: osmo not on PATH after install — check the installer output" >&2 +fi echo "==> Creating directories" install -d -o "$USER_NAME" -g "$USER_NAME" -m 0750 "$INSTALL_DIR" install -d -o root -g "$USER_NAME" -m 0750 "$CONFIG_DIR" install -d -o "$USER_NAME" -g "$USER_NAME" -m 0750 "$STATE_DIR" +# XDG dirs for the osmo CLI login session (see the systemd unit's HOME/XDG env). +install -d -o "$USER_NAME" -g "$USER_NAME" -m 0700 "$STATE_DIR/.config" +install -d -o "$USER_NAME" -g "$USER_NAME" -m 0700 "$STATE_DIR/.cache" +install -d -o "$USER_NAME" -g "$USER_NAME" -m 0700 "$STATE_DIR/.state" echo "==> Copying orchestrator files to $INSTALL_DIR" install -o "$USER_NAME" -g "$USER_NAME" -m 0755 \ "$REPO_DIR/orchestrator.py" "$INSTALL_DIR/orchestrator.py" install -o "$USER_NAME" -g "$USER_NAME" -m 0644 \ - "$REPO_DIR/cloud-init.yaml.j2" "$INSTALL_DIR/cloud-init.yaml.j2" + "$REPO_DIR/runner-workflow.yaml.j2" "$INSTALL_DIR/runner-workflow.yaml.j2" echo "==> Building Python venv" sudo -u "$USER_NAME" python3 -m venv "$INSTALL_DIR/venv" @@ -63,11 +82,14 @@ fi install -o root -g "$USER_NAME" -m 0640 /tmp/github-pat "$CONFIG_DIR/github-pat" shred -u /tmp/github-pat -echo "==> Verifying clouds.yaml" -CLOUDS_YAML="/home/$USER_NAME/.config/openstack/clouds.yaml" -if [[ ! -f "$CLOUDS_YAML" ]]; then - echo "WARNING: $CLOUDS_YAML missing." >&2 - echo " Create it (application credential) before starting the service." >&2 +echo "==> Installing OSMO service-account token (from /tmp/osmo-token)" +if [[ -f /tmp/osmo-token ]]; then + install -o root -g "$USER_NAME" -m 0640 /tmp/osmo-token "$CONFIG_DIR/osmo-token" + shred -u /tmp/osmo-token +else + echo "WARNING: /tmp/osmo-token not found." >&2 + echo " Stage the OSMO service-account token before starting the service:" >&2 + echo " sudo install -o root -g $USER_NAME -m 0640 /tmp/osmo-token $CONFIG_DIR/osmo-token" >&2 fi echo "==> Installing systemd unit" @@ -78,7 +100,8 @@ systemctl daemon-reload echo echo "Setup complete. Next steps:" -echo " 1. Edit $CONFIG_DIR/config.yaml — fill flavor/network/keypair/security_group." -echo " 2. Verify $CLOUDS_YAML exists with the application credential." +echo " 1. Edit $CONFIG_DIR/config.yaml — set osmo_url, pool, platform," +echo " runner_image, and resources." +echo " 2. Ensure $CONFIG_DIR/osmo-token holds the OSMO service-account token." echo " 3. systemctl enable --now airstack-orchestrator.service" echo " 4. journalctl -u airstack-orchestrator.service -f" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index add5f5953..403864781 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -12,6 +12,11 @@ on: required: false default: 'desktop,isaac-sim,ms-airsim' type: string + force_rebuild: + description: 'Force a full rebuild of every service (skip retag)' + required: false + default: false + type: boolean env: DEFAULT_PROFILES: 'desktop,isaac-sim,ms-airsim' @@ -21,6 +26,8 @@ jobs: runs-on: ubuntu-latest outputs: tag-changed: ${{ steps.check-changes.outputs.tag-changed }} + current-version: ${{ steps.check-changes.outputs.current-version }} + previous-version: ${{ steps.check-changes.outputs.previous-version }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -33,20 +40,22 @@ jobs: run: | # Get the current VERSION value CURRENT_TAG=$(grep "^VERSION=" .env | cut -d '=' -f2- | tr -d '"' | tr -d "'") - + # Get the previous VERSION value git show HEAD~1:.env > .env.prev 2>/dev/null || echo "" > .env.prev PREVIOUS_TAG=$(grep "^VERSION=" .env.prev | cut -d '=' -f2- | tr -d '"' | tr -d "'" || echo "") - + echo "Current tag: $CURRENT_TAG" echo "Previous tag: $PREVIOUS_TAG" - + echo "current-version=$CURRENT_TAG" >> "$GITHUB_OUTPUT" + echo "previous-version=$PREVIOUS_TAG" >> "$GITHUB_OUTPUT" + if [ "$CURRENT_TAG" != "$PREVIOUS_TAG" ] && [ -n "$CURRENT_TAG" ]; then echo "VERSION has changed from '$PREVIOUS_TAG' to '$CURRENT_TAG'" - echo "tag-changed=true" >> $GITHUB_OUTPUT + echo "tag-changed=true" >> "$GITHUB_OUTPUT" else echo "VERSION has not changed" - echo "tag-changed=false" >> $GITHUB_OUTPUT + echo "tag-changed=false" >> "$GITHUB_OUTPUT" fi docker-build: @@ -78,7 +87,6 @@ jobs: - name: Verify .env file and extract tag run: | - # Ensure .env file exists and is readable if [ ! -f .env ]; then echo "Error: .env file not found" exit 1 @@ -89,68 +97,184 @@ jobs: # Some compose files expect this file to exist, even if it is empty. mkdir -p simulation/isaac-sim/docker : > simulation/isaac-sim/docker/omni_pass.env - - # Display the current VERSION for debugging + DOCKER_TAG=$(grep "^VERSION=" .env | cut -d '=' -f2- | tr -d '"' | tr -d "'") echo "Building with VERSION: $DOCKER_TAG" - + if [ -z "$DOCKER_TAG" ]; then echo "Error: VERSION is empty" exit 1 fi - - name: Run Docker Compose Build + - name: Resolve compose profiles and previous VERSION + id: prep run: | - # Load environment variables and run docker compose build - set -a # Export all variables + set -a source .env - set +a # Stop exporting - - # Always override COMPOSE_PROFILES for all trigger types + set +a + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" else export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" fi - - docker compose build - - name: Run Docker Compose Push + PREVIOUS_VERSION="${{ needs.check-docker-tag-change.outputs.previous-version }}" + CURRENT_VERSION="${{ needs.check-docker-tag-change.outputs.current-version }}" + if [ -z "$CURRENT_VERSION" ]; then + CURRENT_VERSION=$(grep "^VERSION=" .env | cut -d '=' -f2- | tr -d '"' | tr -d "'") + fi + + FORCE_REBUILD=false + if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ github.event.inputs.force_rebuild }}" == "true" ]; then + FORCE_REBUILD=true + fi + + echo "COMPOSE_PROFILES=$COMPOSE_PROFILES" >> "$GITHUB_ENV" + echo "PREVIOUS_VERSION=$PREVIOUS_VERSION" >> "$GITHUB_ENV" + echo "CURRENT_VERSION=$CURRENT_VERSION" >> "$GITHUB_ENV" + echo "FORCE_REBUILD=$FORCE_REBUILD" >> "$GITHUB_ENV" + echo "profiles=$COMPOSE_PROFILES" >> "$GITHUB_OUTPUT" + echo "previous-version=$PREVIOUS_VERSION" >> "$GITHUB_OUTPUT" + echo "force-rebuild=$FORCE_REBUILD" >> "$GITHUB_OUTPUT" + + # Content-aware publish: retag previous versioned images when image inputs + # are unchanged; rebuild only services whose fingerprint differs (or when + # force_rebuild / unlabeled previous images force a cold build). + - name: Plan retag vs rebuild + id: plan run: | - # Load environment variables and run docker compose push - set -a # Export all variables + set -a source .env - set +a # Stop exporting - - # Always override COMPOSE_PROFILES for all trigger types - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" - else - export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" + set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" + + FORCE_ARGS=() + if [ "${{ env.FORCE_REBUILD }}" = "true" ]; then + FORCE_ARGS+=(--force-rebuild) fi - - docker compose push - - name: Sign pushed images with Cosign (keyless) - env: - COSIGN_YES: "true" + python3 .github/workflows/scripts/docker_image_plan.py \ + --version "${{ env.CURRENT_VERSION }}" \ + --previous-version "${{ env.PREVIOUS_VERSION }}" \ + --profiles "${{ env.COMPOSE_PROFILES }}" \ + --plan-out docker-image-plan.json \ + --override-out docker-compose.fingerprint.yaml \ + "${FORCE_ARGS[@]}" + + echo "Plan summary:" + jq -r '.services | to_entries[] | " \(.key): \(.value.action) (\(.value.reason))"' docker-image-plan.json + + - name: Retag unchanged images + run: | + set -euo pipefail + RETAG_COUNT=$(jq '[.services[] | select(.action=="retag")] | length' docker-image-plan.json) + echo "Services to retag: $RETAG_COUNT" + if [ "$RETAG_COUNT" -eq 0 ]; then + echo "Nothing to retag." + exit 0 + fi + + jq -c '.services | to_entries[] | select(.value.action=="retag") | .value' docker-image-plan.json \ + | while IFS= read -r row; do + IMAGE=$(echo "$row" | jq -r '.image') + PREV=$(echo "$row" | jq -r '.previous_image') + CACHE=$(echo "$row" | jq -r '.cache_tag // empty') + echo "Retagging $PREV → $IMAGE" + CREATE_ARGS=(--tag "$IMAGE") + if [ -n "$CACHE" ] && [ "$CACHE" != "null" ]; then + echo " also → $CACHE" + CREATE_ARGS+=(--tag "$CACHE") + fi + docker buildx imagetools create "${CREATE_ARGS[@]}" "$PREV" + done + + # Build/push one service at a time so a single Dockerfile failure (e.g. + # isaac-sim PX4 apt) does not discard successful siblings before push. + - name: Build and push changed images + id: build_push run: | + set -uo pipefail set -a source .env set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" - else - export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" + mapfile -t BUILD_SERVICES < <(jq -r '.services | to_entries[] | select(.value.action=="build") | .key' docker-image-plan.json | sort) + if [ "${#BUILD_SERVICES[@]}" -eq 0 ]; then + echo "No services require a rebuild." + echo "built_services=" >> "$GITHUB_OUTPUT" + exit 0 fi - IMAGES=$(docker compose config --images | sort -u) - if [ -z "$IMAGES" ]; then - echo "No images resolved from compose config; nothing to sign." + echo "Building services sequentially: ${BUILD_SERVICES[*]}" + FAILED=() + BUILT=() + for SVC in "${BUILD_SERVICES[@]}"; do + echo "::group::Build $SVC" + if docker compose \ + -f docker-compose.yaml \ + -f docker-compose.fingerprint.yaml \ + build "$SVC"; then + echo "::endgroup::" + echo "::group::Push $SVC" + if docker compose \ + -f docker-compose.yaml \ + -f docker-compose.fingerprint.yaml \ + push "$SVC"; then + CACHE=$(jq -r --arg s "$SVC" '.services[$s].cache_tag // empty' docker-image-plan.json) + if [ -n "$CACHE" ]; then + echo "Pushing cache tag $CACHE" + docker push "$CACHE" || echo "::warning::Failed to push cache tag $CACHE" + fi + BUILT+=("$SVC") + else + echo "::error::Push failed for $SVC" + FAILED+=("$SVC") + fi + echo "::endgroup::" + else + echo "::endgroup::" + echo "::error::Build failed for $SVC" + FAILED+=("$SVC") + fi + done + + echo "built_services=${BUILT[*]}" >> "$GITHUB_OUTPUT" + if [ "${#FAILED[@]}" -gt 0 ]; then + echo "Failed services: ${FAILED[*]}" exit 1 fi + # Sign whatever was published even if a sibling service build failed. + - name: Sign published images with Cosign (keyless) + if: always() && steps.plan.outcome == 'success' + env: + COSIGN_YES: "true" + run: | + set -euo pipefail + set -a + source .env + set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" + + # Retagged services are always published; rebuilt ones only if push succeeded. + BUILT_CSV="${{ steps.build_push.outputs.built_services }}" + IMAGES=$( + { + jq -r '.services | to_entries[] | select(.value.action=="retag") | .value.image' docker-image-plan.json + if [ -n "$BUILT_CSV" ]; then + for SVC in $BUILT_CSV; do + jq -r --arg s "$SVC" '.services[$s].image // empty' docker-image-plan.json + done + fi + } | awk 'NF' | sort -u + ) + if [ -z "$IMAGES" ]; then + echo "No published images to sign." + exit 0 + fi + for IMG in $IMAGES; do DIGEST=$(docker buildx imagetools inspect "$IMG" --format '{{.Manifest.Digest}}') if [ -z "$DIGEST" ]; then @@ -164,18 +288,25 @@ jobs: done - name: Verify Cosign signatures + if: always() && steps.plan.outcome == 'success' run: | + set -euo pipefail set -a source .env set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" - else - export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" - fi - - IMAGES=$(docker compose config --images | sort -u) + BUILT_CSV="${{ steps.build_push.outputs.built_services }}" + IMAGES=$( + { + jq -r '.services | to_entries[] | select(.value.action=="retag") | .value.image' docker-image-plan.json + if [ -n "$BUILT_CSV" ]; then + for SVC in $BUILT_CSV; do + jq -r --arg s "$SVC" '.services[$s].image // empty' docker-image-plan.json + done + fi + } | awk 'NF' | sort -u + ) for IMG in $IMAGES; do DIGEST=$(docker buildx imagetools inspect "$IMG" --format '{{.Manifest.Digest}}') REPO="${IMG%:*}" @@ -187,13 +318,21 @@ jobs: > /dev/null done - - name: Optional - Run Docker Compose Up (uncomment if needed) + - name: Fail job if any service build/push failed + if: always() && steps.build_push.outcome == 'failure' run: | - # Uncomment the following lines if you also want to start the services - # set -a - # source .env - # set +a - # docker compose up -d + echo "One or more services failed to build or push (see Build and push changed images)." + exit 1 + + - name: Upload image plan artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: docker-image-plan-${{ github.run_id }} + path: | + docker-image-plan.json + docker-compose.fingerprint.yaml + if-no-files-found: ignore notify: needs: [check-docker-tag-change, docker-build] @@ -203,8 +342,8 @@ jobs: - name: Notify build and push result run: | if [ "${{ needs.docker-build.result }}" == "success" ]; then - echo "✅ Docker Compose build and push completed successfully" + echo "✅ Docker Compose build/retag and push completed successfully" else - echo "❌ Docker Compose build or push failed" + echo "❌ Docker Compose build/retag or push failed" exit 1 - fi \ No newline at end of file + fi diff --git a/.github/workflows/scripts/docker_image_plan.py b/.github/workflows/scripts/docker_image_plan.py new file mode 100755 index 000000000..dcaf1a7ae --- /dev/null +++ b/.github/workflows/scripts/docker_image_plan.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python3 +"""Plan retag-vs-rebuild for AirStack docker-build.yml publishes. + +For each compose service with a ``build:`` section under the selected profiles, +compute a content fingerprint of its image inputs. If the previous versioned +image carries the same ``org.airstack.content-fingerprint`` label, the service +is marked ``retag``; otherwise ``build``. + +Also writes an ephemeral compose override that applies the fingerprint as a +build label on services that will be rebuilt. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +FINGERPRINT_LABEL = "org.airstack.content-fingerprint" + +# Explicit roots so broad compose ``context:`` dirs do not hash the whole tree. +# Keys are compose service names after ``docker compose config`` resolution. +SERVICE_FINGERPRINT_ROOTS: dict[str, list[str]] = { + "robot-desktop": [ + "robot/docker/Dockerfile.robot", + "robot/docker/docker-compose.yaml", + "robot/docker/robot-base-docker-compose.yaml", + "robot/docker/custom_rosdep.yaml", + "robot/docker/wait_for_px4.py", + "robot/docker/.bashrc", + "robot/docker/robot_name_map", + ], + "gcs": [ + "gcs/docker/Dockerfile.gcs", + "gcs/docker/docker-compose.yaml", + "gcs/docker/gcs-base-docker-compose.yaml", + "gcs/docker/.bashrc", + "gcs/docker/resources", + "gcs/docker/Foxglove", + ], + "isaac-sim": [ + "simulation/isaac-sim/docker/Dockerfile.isaac-ros", + "simulation/isaac-sim/docker/docker-compose.yaml", + "simulation/isaac-sim/docker/fastdds.xml", + "simulation/isaac-sim/docker/.bashrc", + "simulation/isaac-sim/docker/omniverse.toml", + ], + "ms-airsim": [ + "simulation/ms-airsim/docker/Dockerfile", + "simulation/ms-airsim/docker/docker-compose.yaml", + "simulation/ms-airsim/docker/entrypoint.sh", + ], +} + +# Extra roots when DOCKER_IMAGE_BUILD_MODE=prebuilt (workspace baked into image). +PREBUILT_EXTRA_ROOTS: dict[str, list[str]] = { + "robot-desktop": [ + "robot/ros_ws/src", + "common/ros_packages", + "common/fastdds.xml", + ], + "gcs": [ + "gcs/ros_ws", + "common/ros_packages", + ], +} + +# .env keys whose values affect image tags or layers (included in fingerprint). +ENV_FINGERPRINT_KEYS = ( + "DOCKER_IMAGE_BUILD_MODE", + "PROJECT_DOCKER_REGISTRY", + "PROJECT_NAME", + "CACHE_TAG", +) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def parse_env_value(raw: str) -> str: + """Parse a .env value, honoring quotes and stripping trailing comments.""" + raw = raw.strip() + if not raw: + return "" + if raw[0] in "\"'": + quote = raw[0] + end = raw.find(quote, 1) + if end != -1: + return raw[1:end] + return raw[1:] + # Unquoted: drop an inline ` # comment` (space-hash) or a leading `#`. + if " #" in raw: + raw = raw.split(" #", 1)[0].rstrip() + return raw.strip().strip('"').strip("'") + + +def load_dotenv(path: Path) -> dict[str, str]: + env: dict[str, str] = {} + if not path.is_file(): + return env + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + env[key.strip()] = parse_env_value(value) + return env + + +def run(cmd: list[str], *, cwd: Path, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + cwd=cwd, + check=check, + text=True, + capture_output=True, + ) + + +def compose_config(root: Path, profiles: str, env: dict[str, str]) -> dict[str, Any]: + cmd_env = os.environ.copy() + cmd_env.update(env) + cmd_env["COMPOSE_PROFILES"] = profiles + proc = subprocess.run( + ["docker", "compose", "-f", "docker-compose.yaml", "config", "--format", "json"], + cwd=root, + env=cmd_env, + text=True, + capture_output=True, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + "docker compose config failed:\n" + f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + return json.loads(proc.stdout) + + +def git_ls_files(root: Path, pathspec: str) -> list[str]: + proc = run( + ["git", "ls-files", "-z", "--", pathspec], + cwd=root, + check=False, + ) + if proc.returncode != 0: + return [] + return [p for p in proc.stdout.split("\0") if p] + + +def collect_tracked_files(root: Path, roots: list[str]) -> list[str]: + files: set[str] = set() + for rel in roots: + path = root / rel + if path.is_file(): + files.add(rel) + continue + if path.is_dir(): + for tracked in git_ls_files(root, rel): + # Skip local secrets / generated pass files + if tracked.endswith("omni_pass.env"): + continue + if "/.dev/" in f"/{tracked}/" or tracked.endswith("/.dev"): + continue + files.add(tracked) + return sorted(files) + + +def file_sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def resolve_dockerfile(root: Path, service_cfg: dict[str, Any]) -> Path | None: + build = service_cfg.get("build") or {} + if not isinstance(build, dict): + return None + dockerfile = build.get("dockerfile") + context = build.get("context") or "." + if not dockerfile: + return None + # compose config usually resolves dockerfile to an absolute path + df = Path(dockerfile) + if df.is_absolute(): + return df + return (Path(context) / df).resolve() if Path(context).is_absolute() else (root / context / df).resolve() + + +def find_dockerignore(dockerfile: Path, context: Path) -> Path | None: + for candidate in ( + context / ".dockerignore", + dockerfile.parent / ".dockerignore", + ): + if candidate.is_file(): + return candidate + return None + + +def previous_image_ref(image: str, version: str, previous_version: str) -> str | None: + if not previous_version or not version or version == previous_version: + return None + # Tags look like ...:v{VERSION}_suffix — replace only the version segment. + needle = f":v{version}_" + if needle not in image: + # Fallback: replace first occurrence of the bare version in the tag. + tag_part = image.rsplit(":", 1) + if len(tag_part) != 2 or version not in tag_part[1]: + return None + return f"{tag_part[0]}:{tag_part[1].replace(version, previous_version, 1)}" + return image.replace(needle, f":v{previous_version}_", 1) + + +def cache_tag_from_build(build: dict[str, Any], cache_tag: str) -> str | None: + tags = build.get("tags") or [] + pfx = f":{cache_tag}_" + for tag in tags: + if isinstance(tag, str) and pfx in tag: + return tag + return None + + +def inspect_fingerprint_label(image: str) -> str | None: + """Return the fingerprint label from a registry image, or None if unavailable.""" + proc = subprocess.run( + [ + "docker", + "buildx", + "imagetools", + "inspect", + image, + "--format", + "{{json .}}", + ], + text=True, + capture_output=True, + check=False, + ) + if proc.returncode != 0: + return None + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + return None + + # buildx JSON shape varies by version; hunt for Labels in common places. + def walk(obj: Any) -> str | None: + if isinstance(obj, dict): + labels = obj.get("Labels") or obj.get("labels") + if isinstance(labels, dict) and FINGERPRINT_LABEL in labels: + return labels[FINGERPRINT_LABEL] + for v in obj.values(): + found = walk(v) + if found: + return found + elif isinstance(obj, list): + for item in obj: + found = walk(item) + if found: + return found + return None + + return walk(data) + + +def compute_fingerprint( + root: Path, + service_name: str, + service_cfg: dict[str, Any], + env: dict[str, str], +) -> str: + build = service_cfg.get("build") or {} + roots = list(SERVICE_FINGERPRINT_ROOTS.get(service_name, [])) + + dockerfile = resolve_dockerfile(root, service_cfg) + if dockerfile and dockerfile.is_file(): + try: + rel = str(dockerfile.relative_to(root)) + except ValueError: + rel = str(dockerfile) + if rel not in roots: + roots.insert(0, rel) + + mode = env.get("DOCKER_IMAGE_BUILD_MODE", "dev") + if mode == "prebuilt": + roots.extend(PREBUILT_EXTRA_ROOTS.get(service_name, [])) + + # If service has no map entry, fall back to dockerfile directory. + if not roots and dockerfile is not None: + try: + roots = [str(dockerfile.parent.relative_to(root))] + except ValueError: + roots = [] + + tracked = collect_tracked_files(root, roots) + + h = hashlib.sha256() + h.update(f"service:{service_name}\n".encode()) + h.update(f"DOCKER_IMAGE_BUILD_MODE:{mode}\n".encode()) + + for key in ENV_FINGERPRINT_KEYS: + h.update(f"env:{key}={env.get(key, '')}\n".encode()) + + args = build.get("args") or {} + if isinstance(args, dict): + for k in sorted(args): + h.update(f"arg:{k}={args[k]}\n".encode()) + + context = build.get("context") + if context: + h.update(f"context:{context}\n".encode()) + ctx_path = Path(context) if Path(context).is_absolute() else root / context + dockerignore = find_dockerignore(dockerfile, ctx_path) if dockerfile else None + if dockerignore and dockerignore.is_file(): + h.update(f"dockerignore:{dockerignore.name}\n".encode()) + h.update(dockerignore.read_bytes()) + h.update(b"\n") + + for rel in tracked: + path = root / rel + if not path.is_file(): + continue + h.update(f"file:{rel}\n".encode()) + h.update(file_sha256(path).encode()) + h.update(b"\n") + + return h.hexdigest() + + +def write_override(path: Path, build_services: dict[str, str]) -> None: + """Write compose override that sets build.labels fingerprint for rebuilds.""" + lines = [ + "# Generated by docker_image_plan.py — do not commit.", + "services:", + ] + if not build_services: + # Valid empty mapping; compose merge ignores it when nothing rebuilds. + lines[-1] = "services: {}" + else: + for name, fingerprint in sorted(build_services.items()): + lines.append(f" {name}:") + lines.append(" build:") + lines.append(" labels:") + lines.append(f" {FINGERPRINT_LABEL}: \"{fingerprint}\"") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def build_plan( + root: Path, + *, + version: str, + previous_version: str, + profiles: str, + force_rebuild: bool, + env: dict[str, str], +) -> dict[str, Any]: + config = compose_config(root, profiles, env) + services_out: dict[str, Any] = {} + cache_tag = env.get("CACHE_TAG") or "cache" + + for name, cfg in (config.get("services") or {}).items(): + if not isinstance(cfg, dict): + continue + build = cfg.get("build") + if not isinstance(build, dict) or not build: + continue + + image = cfg.get("image") + if not isinstance(image, str) or not image: + continue + + fingerprint = compute_fingerprint(root, name, cfg, env) + prev_image = previous_image_ref(image, version, previous_version) + cache_image = cache_tag_from_build(build, cache_tag) + + action = "build" + reason = "force_rebuild" if force_rebuild else "default_build" + prev_fp = None + if force_rebuild: + action = "build" + reason = "force_rebuild" + elif not prev_image: + action = "build" + reason = "no_previous_image" + else: + prev_fp = inspect_fingerprint_label(prev_image) + if prev_fp is None: + action = "build" + reason = "previous_missing_or_unlabeled" + elif prev_fp == fingerprint: + action = "retag" + reason = "fingerprint_match" + else: + action = "build" + reason = "fingerprint_mismatch" + + services_out[name] = { + "image": image, + "previous_image": prev_image, + "cache_tag": cache_image, + "fingerprint": fingerprint, + "previous_fingerprint": prev_fp, + "action": action, + "reason": reason, + } + + return { + "previous_version": previous_version, + "version": version, + "profiles": profiles, + "force_rebuild": force_rebuild, + "label": FINGERPRINT_LABEL, + "services": services_out, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=None, help="Repo root (default: AirStack/)") + parser.add_argument("--version", default="", help="Current VERSION (default: from .env)") + parser.add_argument("--previous-version", default="", help="Previous VERSION for retag source") + parser.add_argument( + "--profiles", + default="", + help="Compose profiles (default: COMPOSE_PROFILES or desktop,isaac-sim,ms-airsim)", + ) + parser.add_argument( + "--force-rebuild", + action="store_true", + help="Mark every service as build", + ) + parser.add_argument( + "--plan-out", + type=Path, + default=Path("docker-image-plan.json"), + help="Where to write the plan JSON", + ) + parser.add_argument( + "--override-out", + type=Path, + default=Path("docker-compose.fingerprint.yaml"), + help="Compose override with build labels for rebuild services", + ) + parser.add_argument( + "--github-output", + type=Path, + default=None, + help="Optional path to append GITHUB_OUTPUT keys", + ) + args = parser.parse_args(argv) + + root = (args.root or repo_root()).resolve() + env = load_dotenv(root / ".env") + # Prefer process env overlays (CI exports .env via set -a). + for key in ( + "VERSION", + "DOCKER_IMAGE_BUILD_MODE", + "PROJECT_DOCKER_REGISTRY", + "PROJECT_NAME", + "CACHE_TAG", + "COMPOSE_PROFILES", + ): + if os.environ.get(key): + env[key] = os.environ[key] + + version = args.version or env.get("VERSION") or "" + if not version: + print("ERROR: VERSION is empty", file=sys.stderr) + return 1 + + previous_version = args.previous_version + profiles = ( + args.profiles + or os.environ.get("COMPOSE_PROFILES") + or env.get("COMPOSE_PROFILES") + or "desktop,isaac-sim,ms-airsim" + ) + + plan = build_plan( + root, + version=version, + previous_version=previous_version, + profiles=profiles, + force_rebuild=args.force_rebuild, + env=env, + ) + + build_services = { + name: svc["fingerprint"] + for name, svc in plan["services"].items() + if svc["action"] == "build" + } + retag_services = [name for name, svc in plan["services"].items() if svc["action"] == "retag"] + + args.plan_out = args.plan_out if args.plan_out.is_absolute() else root / args.plan_out + args.override_out = ( + args.override_out if args.override_out.is_absolute() else root / args.override_out + ) + args.plan_out.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_override(args.override_out, build_services) + + print(f"Wrote plan → {args.plan_out}") + print(f"Wrote override → {args.override_out}") + for name, svc in sorted(plan["services"].items()): + print( + f" {name}: action={svc['action']} reason={svc['reason']} " + f"fp={svc['fingerprint'][:12]}…" + ) + + build_list = " ".join(sorted(build_services)) + retag_list = " ".join(sorted(retag_services)) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as fh: + fh.write(f"plan_path={args.plan_out}\n") + fh.write(f"override_path={args.override_out}\n") + fh.write(f"build_services={build_list}\n") + fh.write(f"retag_services={retag_list}\n") + fh.write(f"build_count={len(build_services)}\n") + fh.write(f"retag_count={len(retag_services)}\n") + else: + # Also support GITHUB_OUTPUT env when set by Actions. + gh_out = os.environ.get("GITHUB_OUTPUT") + if gh_out: + with open(gh_out, "a", encoding="utf-8") as fh: + fh.write(f"plan_path={args.plan_out}\n") + fh.write(f"override_path={args.override_out}\n") + fh.write(f"build_services={build_list}\n") + fh.write(f"retag_services={retag_list}\n") + fh.write(f"build_count={len(build_services)}\n") + fh.write(f"retag_count={len(retag_services)}\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 3ddebda58..1ce390ebe 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -152,16 +152,28 @@ jobs: print(f'::error::Could not parse pytest args from comment: {e}', file=sys.stderr) sys.exit(1) + # CI-only flag: do not forward to pytest. + no_image_build = False + stripped = [] + for a in args: + if a in ('--no-image-build', '--pull-only'): + no_image_build = True + else: + stripped.append(a) + args = stripped + # Pull out --sim and -m so the image-prep step can scope profiles # and decide whether to skip (build_docker tests rebuild themselves). # When --sim isn't given we mirror conftest's default so prep covers # whatever pytest will actually exercise. sim = 'msairsim,isaacsim' + sim_explicit = False marks = '' marks_idx = -1 for i, a in enumerate(args): if a == '--sim' and i + 1 < len(args): sim = args[i + 1] + sim_explicit = True elif a == '-m' and i + 1 < len(args): marks = args[i + 1] marks_idx = i + 1 @@ -174,6 +186,24 @@ jobs: marks = f'build_packages or {marks}' args[marks_idx] = marks + # colcon tests do not need Isaac. Default --sim would otherwise bake + # isaac-sim + ms-airsim (~1h) before a 1s colcon test. Pull registry + # cache tags instead; never image-build. + marks_norm = marks.replace('"', '').replace("'", '').strip() + args_blob = ' '.join(args) + heavy = any(m in marks_norm for m in ( + 'liveliness', 'sensors', 'takeoff_hover_land', 'autonomy', 'build_docker', + )) + only_packages = marks_norm == 'build_packages' or ( + not heavy and any(s in args_blob for s in ( + 'test_build_packages', 'test_colcon_', + )) + ) + if only_packages: + no_image_build = True + if not sim_explicit: + sim = 'msairsim' + skip_prep = 'build_docker' in marks quoted = ' '.join(shlex.quote(a) for a in args) @@ -181,10 +211,12 @@ jobs: f.write(f'pytest_args={quoted}\n') f.write(f'sim={sim}\n') f.write(f'skip_image_prep={"true" if skip_prep else "false"}\n') + f.write(f'no_image_build={"true" if no_image_build else "false"}\n') print(f'Resolved pytest args: {quoted or "(none — pytest defaults)"}') print(f'Resolved sim profile: {sim}') print(f'Skip image prep: {skip_prep}') + print(f'No image build (pull/retag only): {no_image_build}') PYEOF # Reply on the PR thread so the commenter sees their /pytest was @@ -200,7 +232,10 @@ jobs: const args = ${{ toJSON(steps.parse.outputs.pytest_args) }}; const cmd = `pytest tests/ ${args}`.trim(); const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const note = `Note: \`build_packages\` is automatically prepended whenever any marks are specified, to ensure code is built before launch tests run.`; + const pullOnly = '${{ steps.parse.outputs.no_image_build }}' === 'true'; + const note = pullOnly + ? `Note: pull-only image prep (no \`image-build\`). \`-m build_packages\` does not pull Isaac Sim. Add \`--no-image-build\` on other marks to skip rebuilds.` + : `Note: \`build_packages\` is automatically prepended whenever any marks are specified, to ensure code is built before launch tests run.`; await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, @@ -268,6 +303,10 @@ jobs: # inline cache (build_docker tests get layer-reuse speedup) and pre-pulls # before `airstack up` (other tests skip the implicit rebuild). When # secrets are absent both steps are skipped and behavior is unchanged. + # + # Read-only on purpose: AIRSTACK_REGISTRY_CACHE_PUSH stays unset here so a + # PR can consume the floating cache tag but never republish it. Only + # docker-build.yml (main/develop) writes it. - name: Log in to internal Docker registry id: docker_login if: ${{ vars.DOCKER_REGISTRY_URL != '' && env.DOCKER_REGISTRY_PASSWORD != '' }} @@ -284,26 +323,30 @@ jobs: - name: Ensure airstack.sh is executable run: chmod +x airstack.sh + - name: Disable compose image builds + if: ${{ steps.parse.outputs.no_image_build == 'true' }} + run: echo "AIRSTACK_NO_IMAGE_BUILD=1" >> "$GITHUB_ENV" + # The ephemeral runner starts with no local images. `airstack_env` in # tests/conftest.py fails fast if compose images are missing, so prep # them here. Profile-gated services (ms-airsim, isaac-sim) are skipped # by compose unless their profile is active, so we mirror the fixture's - # profile selection from the parsed --sim. Pull-only by default; fall - # back to a full build only if the registry doesn't have everything - # (e.g. new branch with no published image yet). Skipped when the - # marks expression contains build_docker — those tests build per-service - # themselves. + # profile selection from the parsed --sim. Pull versioned tags, then + # retag floating cache_* tags onto the VERSION name (PR tags never + # exist). Fall back to image-build only when --no-image-build is off. + # Skipped when marks contain build_docker — those tests build themselves. - name: Ensure Docker images present if: ${{ steps.parse.outputs.skip_image_prep != 'true' }} env: AIRSTACK_ROOT: ${{ github.workspace }} SIM_INPUT: ${{ steps.parse.outputs.sim }} + NO_IMAGE_BUILD: ${{ steps.parse.outputs.no_image_build }} run: | profiles=desktop [[ ",$SIM_INPUT," == *,msairsim,* ]] && profiles="$profiles,ms-airsim" [[ ",$SIM_INPUT," == *,isaacsim,* ]] && profiles="$profiles,isaac-sim" export COMPOSE_PROFILES="$profiles" - echo "Pulling images for COMPOSE_PROFILES=$COMPOSE_PROFILES" + echo "Pulling images for COMPOSE_PROFILES=$COMPOSE_PROFILES (no_image_build=$NO_IMAGE_BUILD)" # Pull from registry; tolerate per-image failures so we can detect # what's still missing afterwards instead of aborting on the first @@ -311,6 +354,25 @@ jobs: # still surface on stderr. ./airstack.sh --progress=quiet image-pull --ignore-pull-failures || true + # VERSION tags miss on every PR. Seed from floating cache_* tags. + cache_tag="$(grep -E '^CACHE_TAG=' .env 2>/dev/null | cut -d= -f2 | tr -d '"' || true)" + cache_tag="${cache_tag:-cache}" + while IFS= read -r img; do + [[ -z "$img" ]] && continue + if docker image inspect "$img" --format '{{.Id}}' >/dev/null 2>&1; then + continue + fi + # Replace :v_ with :_ (PR versioned tags never exist) + cache_img="$(python3 -c "import re,sys; print(re.sub(r':v[^_]+_', f':{sys.argv[2]}_', sys.argv[1], count=1))" "$img" "$cache_tag")" + echo "Versioned tag missing; trying cache tag $cache_img" + if docker pull --quiet "$cache_img"; then + docker tag "$cache_img" "$img" + echo "Retagged $cache_img -> $img" + else + echo "Cache tag pull failed for $cache_img" + fi + done < <(docker compose -f docker-compose.yaml config --images) + missing=() while IFS= read -r img; do [[ -z "$img" ]] && continue @@ -320,11 +382,16 @@ jobs: done < <(docker compose -f docker-compose.yaml config --images) if (( ${#missing[@]} > 0 )); then - echo "Pull did not produce these images; falling back to build:" + echo "Images still missing after pull/retag:" printf ' - %s\n' "${missing[@]}" + if [[ "$NO_IMAGE_BUILD" == "true" ]]; then + echo "::error::Pull-only mode (--no-image-build or -m build_packages) will not image-build. Run /pytest -m build_docker once, or omit --no-image-build." + exit 1 + fi + echo "Falling back to image-build" ./airstack.sh --progress=quiet image-build else - echo "All required images present after pull — skipping build." + echo "All required images present after pull/retag — skipping build." fi - name: Run tests diff --git a/.gitignore b/.gitignore index 4868b5c74..1711ed743 100644 --- a/.gitignore +++ b/.gitignore @@ -94,10 +94,20 @@ simulation/ms-airsim/assets/scenes/* # Test results tests/results/ +# OSMO mission results: local runs of osmo/workspace/mission_runner.py write +# to osmo/results/; `airstack osmo:fetch` downloads to ./osmo-results/. +osmo/results/* +osmo-results/* + # Local-only — embedded sibling repo, not part of this branch common/rayfronts/ # Docker build cache (root-owned subdirs cause permission warnings on `git add`) robot/docker/cache/ + +# Ephemeral outputs from docker_image_plan.py (docker-build.yml) +docker-image-plan.json +docker-compose.fingerprint.yaml + .DS_Store gcs/.DS_Store diff --git a/AGENTS.md b/AGENTS.md index 0a6b86015..856608098 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ AirStack/ ├── tests/ # System tests (pytest) + metrics reporting ├── .github/ │ ├── workflows/ # GitHub Actions CI (system-tests, docker-build, etc.) -│ └── orchestrator/ # OpenStack-backed ephemeral self-hosted runners +│ └── orchestrator/ # OSMO-backed ephemeral self-hosted runners └── .agents/skills/ # Detailed workflow guides for agents ``` @@ -268,16 +268,28 @@ GitHub Actions workflows live in [`.github/workflows/`](.github/workflows/): ### Ephemeral Runner Orchestrator -GPU-required jobs (`runs-on: [self-hosted, airstack-ephemeral]`) execute on **OpenStack VMs spawned per-job and destroyed on completion**. The orchestrator service code lives in [`.github/orchestrator/`](.github/orchestrator/): +GPU-required jobs (`runs-on: [self-hosted, airstack-ephemeral]`) execute on **ephemeral pods scheduled by [NVIDIA OSMO](https://nvidia.github.io/OSMO/) — one per job, destroyed on completion**. The GitHub side is unchanged from the old OpenStack backend (same labels, JIT tokens, fork guard); only the spawn target moved from "create a Nova VM" to "submit an OSMO workflow". The orchestrator service code lives in [`.github/orchestrator/`](.github/orchestrator/): -- [`orchestrator.py`](.github/orchestrator/orchestrator.py) — Python service: spawn loop polls GitHub for queued jobs matching configured runner labels, mints single-use JIT runner tokens, creates an OpenStack server with cloud-init bootstrap; reap loop deletes the server when the job completes (or after `max_job_minutes`) -- [`cloud-init.yaml.j2`](.github/orchestrator/cloud-init.yaml.j2) — bootstraps Docker + nvidia-container-toolkit + GH Actions runner on the worker, registers with the JIT token, runs one job, then `shutdown -h` -- [`config.example.yaml`](.github/orchestrator/config.example.yaml) — flavor / network / keypair / floating-IP pool / runner labels / repo +- [`orchestrator.py`](.github/orchestrator/orchestrator.py) — Python service: spawn loop polls GitHub for queued jobs matching configured runner labels, mints single-use JIT runner tokens, and submits one OSMO workflow per job (`osmo workflow submit`); reap loop cancels the workflow when the job completes (or after `max_job_minutes`), plus an orphan sweep via `osmo workflow list` +- [`runner-workflow.yaml.j2`](.github/orchestrator/runner-workflow.yaml.j2) + [`runner.Dockerfile`](.github/orchestrator/runner.Dockerfile) + [`runner-entrypoint.sh`](.github/orchestrator/runner-entrypoint.sh) — the per-job worker: a **privileged**, GPU-enabled OSMO task (prebaked image) that starts an inner Docker daemon (the tests run `airstack up` = docker compose), registers with the JIT token, runs one job, then exits so OSMO reaps the pod +- [`config.example.yaml`](.github/orchestrator/config.example.yaml) — osmo_url / pool / platform / runner_image / resources / runner labels / repo - [`airstack-orchestrator.service`](.github/orchestrator/airstack-orchestrator.service) + [`setup.sh`](.github/orchestrator/setup.sh) — systemd unit and one-time installer -**Why ephemeral:** clean Docker cache per run, no leaked containers, GitHub PAT and OpenStack credentials only on the orchestrator host (workers receive a single-use JIT token bound to one runner registration). State map at `/var/lib/airstack-orchestrator/state.json`; logs via `journalctl -u airstack-orchestrator.service -f`. +**Why ephemeral:** clean Docker cache per run, no leaked containers; the GitHub PAT and the OSMO service-account token live only on the orchestrator host (workers receive a single-use JIT token bound to one runner registration). CI authenticates to OSMO as a shared, non-personal [service account](https://nvidia.github.io/OSMO/main/deployment_guide/appendix/authentication/service_accounts.html) scoped to a dedicated CI GPU pool, so runs don't consume individuals' quotas. The CI pool's platform must have **"Privileged Mode Allowed"** enabled (docker-in-docker). State map at `/var/lib/airstack-orchestrator/state.json`; logs via `journalctl -u airstack-orchestrator.service -f`. -**Setup, debugging a failed job, and SSH-into-worker procedures:** [`.github/orchestrator/README.md`](.github/orchestrator/README.md) (also exposed as [`tests/ci-cd-orchestrator.md`](tests/ci-cd-orchestrator.md) symlink for the docs site). +**Nested DinD needs a non-overlayfs Docker data-root.** The OSMO pod's root filesystem is overlayfs, and Linux rejects a directory on overlayfs as an overlay `upperdir` (`EINVAL`). A dockerd storing data on the pod rootfs pulls images fine but fails every build step that needs a real mount, with errors that masquerade as `apt-get`/`WORKDIR` failures: + +``` +failed to solve: ... mount source: "overlay", target: ".../buildkit/containerd-overlayfs/cachemounts/...", err: invalid argument +``` + +[`runner-entrypoint.sh`](.github/orchestrator/runner-entrypoint.sh) picks a backend by attempting a real overlay mount, preferring a loopback ext4 image at `/var/lib/docker` (real `overlay2`), then a real filesystem already mounted in the pod, then `fuse-overlayfs`, then `vfs`. Landing on `vfs` means builds will be slow and probably run out of disk — check the `[runner-entrypoint] storage:` line in the job log first when Docker builds misbehave. Details: [orchestrator README → Nested DinD and overlayfs](.github/orchestrator/README.md). + +**Docker layer cache is a floating tag, not the versioned one.** Every compose service lists two `cache_from` entries: the versioned image (`airstack:v${VERSION}_`) and a floating one (`airstack:${CACHE_TAG:-cache}_`). Only the floating tag can ever hit on a PR — `check-version-increment` forces `VERSION` up on every PR, so the versioned tag it builds under has by definition never been pushed. Reading and writing are separate switches: `AIRSTACK_REGISTRY_CACHE=1` (set by `system-tests.yml`) pulls and builds with `BUILDKIT_INLINE_CACHE=1`, while `AIRSTACK_REGISTRY_CACHE_PUSH=1` (set only by `docker-build.yml` on main/develop) also publishes both tags. PR runs stay read-only so an unmerged branch can't poison the shared cache or publish an unreleased version. If you add a service with a `build:` section, give it both entries or its builds will always be cold. + +**Publish retags when image inputs are unchanged.** `docker-build.yml` runs [`.github/workflows/scripts/docker_image_plan.py`](.github/workflows/scripts/docker_image_plan.py) on VERSION bumps: each service gets a content fingerprint (`org.airstack.content-fingerprint`). If the previous versioned image already has that label, the job registry-retags (`imagetools create`) instead of rebuilding; only changed services rebuild (and refresh `cache_*`). Use `workflow_dispatch` with `force_rebuild=true` to rebuild everything. PR `build_docker` tests still perform real builds. + +**Setup, debugging a failed job, and exec-into-worker procedures:** [`.github/orchestrator/README.md`](.github/orchestrator/README.md) (also exposed as [`tests/ci-cd-orchestrator.md`](tests/ci-cd-orchestrator.md) symlink for the docs site). ## Documentation Requirements diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bde19e49..e64a82978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `docker-build.yml` retags unchanged images on VERSION bumps (content fingerprint) instead of always rebuilding; floating `cache_*` tags still seed PR layer cache - `robot-l4t` compose service knobs are now env-overridable (`AUTONOMY_ROLE`, `FCU_URL`, and the rosbag path via `BAG_STORAGE_PATH`); `FCU_URL` unquoted so the literal serial path reaches MAVROS - `zed-l4t` image: ZED SDK 4.2 → 5.2 with the coupled ROS deps (`zed_msgs` 5.2.1, `point_cloud_transport(_plugins)` 4.x, add `backward_ros`) ### Fixed +- Isaac Sim image: PX4 `ubuntu.sh` no longer fails dpkg configure on the NVIDIA base (`ca-certificates` / `software-properties-common`); use `--no-nuttx --no-sim-tools` like ms-airsim +- Robot image: pin `pytest<8.1` and disable `launch_testing` for colcon unit tests so ROS Jazzy's outdated pytest hook does not abort `colcon test` - Robot name resolution now honors a pre-set `ROBOT_NAME` (e.g. injected via docker compose) instead of always overriding it from the container/hostname mapping (`robot/docker/.bashrc`) - Robot name-map catch-all fallback now maps to `unknown_robot` (valid ROS namespace token) instead of `unknown-robot` (`default_robot_name_map.yaml`) - l4t robot image: replace dustynv's `/ros_entrypoint.sh` with a passthrough so its prebuilt source-ROS libs (older `fastcdr`) no longer shadow the apt Jazzy runtime and crash apt-built nodes like MAVROS diff --git a/airstack.sh b/airstack.sh index 1c57c47cb..d42c5b6e1 100755 --- a/airstack.sh +++ b/airstack.sh @@ -824,6 +824,46 @@ function ensure_robot_l4t_stack_base() { run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${_ga[@]}" build "${build_opts[@]}" robot-l4t-stack-base } +# `docker compose push` only publishes each service's `image:`. The floating +# cache tags are declared in `build.tags`, so they need an explicit push. Read +# them back out of the resolved config instead of reconstructing the names here, +# so this stays correct as services are added. +function push_cache_tags() { + local -n _ga="$1" + local -n _sc="$2" + + if ! command -v jq >/dev/null 2>&1; then + log_warn "jq not found; skipping cache-tag push (floating cache will go stale)" + return 0 + fi + + # Service names only — drop any flags that were passed through to the subcommand. + local services=() + for arg in "${_sc[@]}"; do + [[ "$arg" == -* ]] || services+=("$arg") + done + + local tags + tags=$(run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${_ga[@]}" config --format json 2>/dev/null \ + | jq -r --arg svcs "${services[*]}" --arg pfx ":${CACHE_TAG:-cache}_" ' + .services | to_entries[] + | select($svcs == "" or (($svcs | split(" ")) | index(.key))) + | (.value.build.tags // [])[] + | select(contains($pfx)) + ' 2>/dev/null | sort -u) + + if [[ -z "$tags" ]]; then + log_warn "No cache tags resolved from compose config; nothing to publish" + return 0 + fi + + while IFS= read -r tag; do + [[ -n "$tag" ]] || continue + log_info "Pushing cache tag $tag" + docker push "$tag" || log_warn "Failed to push cache tag $tag" + done <<< "$tags" +} + function cmd_up { check_docker @@ -857,7 +897,12 @@ function cmd_up { fi log_info "Starting services..." - run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" up "${subcmd_args[@]}" -d + local up_opts=() + if [[ "${AIRSTACK_NO_IMAGE_BUILD:-}" == "1" ]]; then + log_info "AIRSTACK_NO_IMAGE_BUILD=1 → compose up --no-build" + up_opts+=(--no-build) + fi + run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" up "${up_opts[@]}" "${subcmd_args[@]}" -d log_info "Services brought up successfully" } @@ -877,9 +922,16 @@ function cmd_image_build { # Registry-cache mode (CI / opt-in): pre-pull existing images to seed the # local cache, build with BUILDKIT_INLINE_CACHE=1 so the resulting image - # carries layer-cache metadata, and push so the next run benefits. The - # cache_from declarations in each component compose file make BuildKit - # actually reuse the pulled layers. No-op when the env var is unset. + # carries layer-cache metadata. The cache_from declarations in each + # component compose file make BuildKit actually reuse the pulled layers. + # No-op when the env var is unset. + # + # Reading and publishing the cache are separate switches. A PR bumps VERSION + # (check-version-increment enforces it), so the versioned cache_from entry is + # guaranteed to miss and the floating CACHE_TAG entry is what actually hits. + # PR runs must not write that floating tag: an unmerged branch would poison + # the shared cache and publish an unreleased VERSION. Only trusted branches + # set AIRSTACK_REGISTRY_CACHE_PUSH=1. if [[ "${AIRSTACK_REGISTRY_CACHE:-}" == "1" ]]; then log_info "AIRSTACK_REGISTRY_CACHE=1 → pulling for cache seed..." run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" pull --ignore-pull-failures "${subcmd_args[@]}" || \ @@ -888,9 +940,14 @@ function cmd_image_build { log_info "Building services with BUILDKIT_INLINE_CACHE=1..." run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" build --build-arg BUILDKIT_INLINE_CACHE=1 "${subcmd_args[@]}" - log_info "Pushing built images for next-run cache..." - run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" push --ignore-push-failures "${subcmd_args[@]}" || \ - log_warn "Post-build push encountered failures; future runs may not benefit from cache" + if [[ "${AIRSTACK_REGISTRY_CACHE_PUSH:-}" == "1" ]]; then + log_info "Pushing built images for next-run cache..." + run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" push --ignore-push-failures "${subcmd_args[@]}" || \ + log_warn "Post-build push encountered failures; future runs may not benefit from cache" + push_cache_tags global_args subcmd_args + else + log_info "AIRSTACK_REGISTRY_CACHE_PUSH is not 1 → cache is read-only for this run" + fi else log_info "Building services..." run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" build "${subcmd_args[@]}" diff --git a/docs/development/intermediate/testing/ci_cd.md b/docs/development/intermediate/testing/ci_cd.md index a5afaf265..c20406739 100644 --- a/docs/development/intermediate/testing/ci_cd.md +++ b/docs/development/intermediate/testing/ci_cd.md @@ -1 +1,568 @@ -# CI/CD Pipeline \ No newline at end of file +# CI/CD Pipeline on OSMO + +AirStack's continuous integration runs the **full drone stack** — simulator, +robot autonomy, and GCS — on a GPU for every change. Because that needs a +GPU, a Docker daemon, and a clean filesystem, jobs cannot run on GitHub's +hosted runners and should not run on a shared always-on machine. Instead, a +small orchestrator service watches the GitHub Actions queue and submits one +**ephemeral [NVIDIA OSMO](https://nvidia.github.io/OSMO/) pod per job**. The pod +registers as a single-use GitHub Actions runner, executes exactly one job, and +is destroyed. + +This page documents the whole system: the architecture, the job lifecycle, +what each test suite actually catches, how to trigger and read a run, and how +to fit CI into your day-to-day development loop. + +!!! note "Related pages" + - [`tests/README.md`](../../../../tests/README.md) — the test suite reference: marks, fixtures, metrics, CLI flags. + - [CI/CD Orchestrator](../../../../tests/ci-cd-orchestrator.md) — the lab-admin runbook: pool prerequisites, credential staging, `setup.sh`, rotation, break-glass debugging. + - [AirStack on OSMO](../../../tutorials/airstack_on_osmo.md) — the *interactive* OSMO dev pod (Remote-SSH + Isaac Sim streaming). Different workflow, same compute pool. + +--- + +## The short version + +| Question | Answer | +|---|---| +| Where do CI jobs run? | A fresh GPU pod on the OSMO `airstack` pool, one per job, destroyed after. | +| What triggers a run? | A PR being **opened**, a `/pytest` comment from a maintainer, or manual `workflow_dispatch`. | +| What gets tested? | Docker image builds, `colcon` builds, unit tests, stack liveliness, sensor rates, takeoff/hover/land, fixed-trajectory tracking. | +| How do I see results? | A metrics report comment on the PR, plus the `test-results-*` artifact (`summary.txt`, `results.xml`, `metrics.json`). | +| What fails the build? | Any failed test, **or** a metric regressing more than 20 % against the base branch's last run. | +| Who holds the secrets? | Only the orchestrator host. Workers get a single-use JIT token valid for one registration. | + +--- + +## Architecture + +Three planes, each owning one job. GitHub owns the queue and the logs. The +orchestrator owns the credentials and the job ↔ pod bookkeeping. OSMO owns the +GPU compute and the pod lifecycle. + +```mermaid +flowchart LR + subgraph gh [GitHub] + pr["Pull request / comment / dispatch"] + queue["Actions queue
workflow_job: queued
labels: self-hosted, airstack-ephemeral"] + api["REST API"] + pr --> queue + queue --- api + end + + subgraph orch ["Orchestrator host (no GPU, always on)"] + svc["airstack-orchestrator.service
orchestrator.py"] + spawn["spawn loop — every 15s"] + reap["reap loop — every 30s"] + creds["/etc/airstack-orchestrator
github-pat + osmo-token + config.yaml"] + state["/var/lib/airstack-orchestrator/state.json
job_id → workflow_id"] + svc --> spawn + svc --> reap + svc --- creds + spawn --- state + reap --- state + end + + subgraph osmo ["OSMO airstack pool (GPU, privileged)"] + wf["Workflow gha-runner-JOBID-TS"] + pod["Ephemeral runner pod
airstack-ci-runner image"] + wf --> pod + end + + api -- "poll queued jobs" --> spawn + spawn -- "mint JIT runner config" --> api + spawn -- "osmo workflow submit" --> wf + pod -- "register + long-poll for work" --> api + reap -- "osmo workflow cancel" --> wf +``` + +Key properties that fall out of this shape: + +- **Truly ephemeral.** Every job starts from the prebaked image with an empty Docker cache. No leftover containers, no dangling networks, no "works because the last run left something behind". +- **PAT isolation.** The GitHub PAT never leaves the orchestrator. The pod receives a [JIT runner config](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository) — a base64 blob bound to exactly one runner registration, short-lived. +- **Non-personal OSMO identity.** The orchestrator authenticates with a service-account token scoped to the CI pool, so runs never consume an individual's GPU quota and nothing breaks when someone graduates. +- **Crash-safe.** Every workflow is named `gha-runner--`. The reap loop cancels any active workflow with that prefix that is missing from `state.json`, so a crashed or restarted orchestrator cannot leak pods. + +--- + +## Job lifecycle + +From "you comment `/pytest`" to "the pod is gone", in order: + +```mermaid +sequenceDiagram + autonumber + participant Dev as Developer + participant GH as GitHub Actions + participant Orch as Orchestrator + participant OSMO as OSMO scheduler + participant Pod as Runner pod + + Dev->>GH: open PR / comment /pytest / dispatch + GH->>GH: queue job with labels self-hosted + airstack-ephemeral + Orch->>GH: poll queued jobs (15s) + Orch->>GH: POST generate-jitconfig + GH-->>Orch: encoded_jit_config (single use) + Orch->>Orch: render runner-workflow.yaml.j2 + Orch->>OSMO: osmo workflow submit --pool airstack + OSMO-->>Orch: workflow name, then uuid via query + Orch->>Orch: record job_id to workflow_id in state.json + OSMO->>Pod: schedule privileged GPU pod + Pod->>Pod: start inner dockerd, nvidia-smi check + Pod->>GH: run.sh --jitconfig, register ephemeral runner + GH->>Pod: dispatch the one job + Pod->>Pod: checkout, pull images, pytest + Pod-->>GH: logs, conclusion, results artifact + Pod->>Pod: run.sh exits after one job, task completes + OSMO->>OSMO: tear the pod down + Orch->>GH: poll job status (30s) + GH-->>Orch: completed + Orch->>OSMO: cancel if still live, then drop from state.json +``` + +Two safety nets run on top of the happy path: + +- **Straggler reap.** Any tracked job older than `max_job_minutes` (default 48 h) is force-cancelled regardless of what GitHub reports. +- **Orphan sweep.** Active `gha-runner-*` workflows that are not in `state.json` and are more than two minutes old get cancelled. The two-minute grace window prevents the sweep from racing a submit that has not been recorded yet. + +--- + +## Anatomy of a runner pod + +The worker is a prebaked image — everything the job needs is already in the +layer cache when the pod starts, so a slow `apt-get` can never outlive the JIT +token's validity window. + +```mermaid +flowchart TB + subgraph pod ["OSMO task — privileged, 1 GPU, 8 CPU, 32Gi RAM, 300Gi disk"] + entry["run-ephemeral-runner.sh"] + dockerd["inner dockerd
+ nvidia-container-toolkit"] + runner["actions-runner run.sh --jitconfig"] + subgraph compose ["docker compose stack started by airstack up"] + sim["isaac-sim or ms-airsim"] + robot["robot-desktop x NUM_ROBOTS"] + gcs["gcs"] + end + entry --> dockerd + entry --> runner + runner -- "pytest tests/ runs airstack up" --> dockerd + dockerd --> sim + dockerd --> robot + dockerd --> gcs + end + gpu["Node GPU"] --> dockerd +``` + +| Piece | File | What it contributes | +|---|---|---| +| Image | [`runner.Dockerfile`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner.Dockerfile) | Ubuntu 24.04 + Docker CE + compose/buildx + NVIDIA container toolkit + pinned `actions/runner` (2.334.0) | +| Entrypoint | [`runner-entrypoint.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-entrypoint.sh) | Starts `dockerd`, waits up to 60 s for it, runs `nvidia-smi` as a non-fatal GPU sanity check, then `exec`s `run.sh --jitconfig` | +| Pod shape | [`runner-workflow.yaml.j2`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-workflow.yaml.j2) | Resource request, `privileged: true`, the JIT config and `RUNNER_ALLOW_RUNASROOT` env | +| Sizing | `config.yaml` | `cpu: 8`, `gpu: 1`, `memory: 32Gi`, `storage: 300Gi` — sized for sim + robot + GCS images plus Isaac assets | + +!!! warning "Privileged is mandatory" + The tests run `airstack up`, which is `docker compose`, which needs a Docker + daemon *inside* the pod. That requires the pool's platform to have + **Privileged Mode Allowed** enabled. Without it, submissions are rejected and + `osmo workflow logs` shows `dockerd did not become ready`. + +Build and publish the image with +[`build-and-push.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/build-and-push.sh), +or — if you have no local Docker — submit +[`build-runner-on-osmo.yaml`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/build-runner-on-osmo.yaml), +a one-shot OSMO job that builds the runner image inside an OSMO pod and pushes +it to Harbor. + +--- + +## Triggering a run + +### The three entry points + +| Trigger | When it fires | What it runs | +|---|---|---| +| `pull_request` (`types: [opened]`) | Only when the PR is first opened, and only for same-repo branches | pytest's `conftest` defaults — the full mark set | +| `/pytest` PR comment | Any time, from a user with `OWNER`/`MEMBER`/`COLLABORATOR` association | Whatever args you put on the first line of the comment | +| `workflow_dispatch` | Manual, from the Actions tab | The form inputs: `marks`, `sim`, `num_robots`, `stress_iterations`, `stable_duration`, `baseline_run_id` | + +Pushes to an open PR deliberately do **not** re-trigger. GPU pods are a shared +resource, so re-runs are opt-in via `/pytest`. + +### Comment syntax + +The first line is parsed with `shlex`; everything after it is free-form notes. + +```text +/pytest -m liveliness --sim msairsim --num-robots 1 --stress-iterations 1 + +Checking whether the DDS bridge fix holds under 3 robots — see thread above. +``` + +`-m build_packages` is **pull-only**: it retags floating `cache_*` images onto the PR `VERSION` tag and never runs `image-build` (and does not pull Isaac Sim). Use that when iterating on colcon/pytest failures. For other marks, add `--no-image-build` to skip the bake: + +```text +/pytest -m build_packages +/pytest -m liveliness --sim msairsim --no-image-build +``` + +The workflow replies on the thread with the exact `pytest` command it resolved +and a link to the run, and opens a **Check Run** pinned to the PR head SHA so +comment-triggered runs still show up in the PR's Checks tab. + +!!! tip "`build_packages` is prepended for you" + Whenever you pass `-m`, the workflow rewrites the expression to + `build_packages or `. Launch tests are useless against a stale + `install/` tree, and this removes the most common way to waste a 40-minute + GPU run. It is skipped when you already named `build_packages`, and when you + pass no marks at all (pytest then runs everything anyway). + +### What the job does, step by step + +```mermaid +flowchart TD + a["Resolve PR head — issue_comment only"] --> b["Parse pytest args
prepend build_packages, extract --sim"] + b --> c["Ack comment + open in-progress Check Run"] + c --> d["Checkout PR head with submodules"] + d --> e["Write omni_pass.env — guest Nucleus creds"] + e --> f["Create venv, install tests/requirements.txt"] + f --> g{"Registry secrets present?"} + g -- yes --> h["docker login, set AIRSTACK_REGISTRY_CACHE=1
read-only: PRs never republish the cache"] + g -- no --> i["Skip — build from scratch"] + h --> j{"marks contain build_docker?"} + i --> j + j -- yes --> l["Skip image prep — those tests build themselves"] + j -- no --> k["airstack image-pull for the active profiles
fall back to image-build for anything missing"] + k --> m["pytest tests/ with resolved args"] + l --> m + m --> n["Upload tests/results/ artifact, 90-day retention"] + n --> o["Finalize Check Run with the job conclusion"] + o --> p["report job on ubuntu-latest"] +``` + +The image-prep step is what makes runs on a cold pod tolerable: it pulls the +published images for exactly the compose profiles the selected `--sim` implies, +then falls back to a local build only for images the registry did not have (a +new branch that has not been released yet, for example). + +### Layer cache: the floating `cache_*` tag + +Every pod starts with an empty Docker cache, so `build_docker` is only fast if +BuildKit can import layers from the registry. Each compose service therefore +declares two `cache_from` entries: + +| Entry | Example | Who writes it | +|---|---|---| +| Versioned | `airstack:v0.19.0-alpha.7_isaac-sim` | `docker-build.yml`, per release | +| Floating | `airstack:cache_isaac-sim` | `docker-build.yml`, republished every build | + +The versioned entry alone cannot work on a pull request. `check-version-increment` +requires every PR to raise `VERSION`, so the tag a PR builds under is by +definition one that has never been pushed — the pull misses and the build runs +cold from the first `RUN` layer: + +``` +Image ...airstack:v0.19.0-alpha.7_isaac-sim Pulling +Image ...airstack:v0.19.0-alpha.7_isaac-sim failed to resolve reference +``` + +The floating tag is the one that actually hits. It tracks the newest build from +`main`/`develop` rather than any particular version, so a PR imports the layers +its base branch already produced and rebuilds only what it changed. + +Reading and writing the cache are separate switches, and PR runs get read only: + +- `AIRSTACK_REGISTRY_CACHE=1` — pull to seed, build with `BUILDKIT_INLINE_CACHE=1`. + Set by `system-tests.yml` whenever registry secrets are available. +- `AIRSTACK_REGISTRY_CACHE_PUSH=1` — additionally publish the versioned and + floating tags. Set only by `docker-build.yml`. + +Keeping the write switch off for pull requests means an unmerged branch can +neither poison the shared cache for everyone else nor publish an unreleased +`VERSION` tag. Override the tag name with `CACHE_TAG` (default `cache`) to keep +an experimental cache line separate. + +### Publish path: retag when image inputs are unchanged + +`check-version-increment` forces every PR to raise `VERSION`, including +docs-only changes. On `main`/`develop`, that would otherwise mean a full +multi-hour rebuild of every image for a no-op Docker change. + +[`docker-build.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/docker-build.yml) +therefore plans per service before building: + +1. [`.github/workflows/scripts/docker_image_plan.py`](https://github.com/castacks/AirStack/blob/main/.github/workflows/scripts/docker_image_plan.py) + hashes each service’s Dockerfile, compose-related files, build args, and + tracked fingerprint roots into `org.airstack.content-fingerprint`. +2. It inspects the **previous** versioned image’s label (from `HEAD~1`’s + `VERSION=`). +3. **Match** → registry-side retag with + `docker buildx imagetools create` (new `v${VERSION}_…` tag and floating + `cache_*` tag, same digest — no rebuild). +4. **Mismatch / missing / unlabeled / `force_rebuild`** → `docker compose build` + for that service only, with the fingerprint applied as a build label via an + ephemeral `docker-compose.fingerprint.yaml` override. + +PR `system-tests` / `build_docker` are unchanged: they still run real builds so +Dockerfiles keep being proven. Floating `cache_*` remains the layer-cache seed +for those rebuilds. + +Manual dispatch accepts `force_rebuild=true` to rebuild and relabel everything +(useful the first time after this lands, or to refresh `cache_*` from scratch). + +--- + +## What the pipeline tests, and what that catches + +Tests are selected with pytest marks. Collection order is fixed in +`tests/conftest.py` so cheap and prerequisite suites always run first — a +`colcon` break fails in minutes instead of after a sim bring-up. + +```mermaid +flowchart LR + u["unit
seconds, no Docker"] --> bd["build_docker
image builds"] + bd --> bp["build_packages
colcon build in containers"] + bp --> lv["liveliness
stack comes up"] + lv --> sn["sensors
streams flow at rate"] + sn --> th["takeoff_hover_land
flight chain"] + th --> au["autonomy
trajectory tracking"] +``` + +| Mark | Module | What it verifies | Bugs it is good at catching | +|---|---|---|---| +| `unit` | `tests/robot/`, `tests/sim/` proxies | Hermetic Python/numpy logic co-located with each ROS 2 package | Off-by-one and boundary errors in filters, converters, validators; regressions in pure algorithm code | +| `build_docker` | `system/test_build_docker.py` | Every image builds; records image sizes | Broken Dockerfiles, deleted apt packages, upstream base-image drift, accidental image bloat | +| `build_packages` | `system/test_build_packages.py` | `colcon build` inside robot, GCS, and ms-airsim workspaces | Missing `package.xml` dependencies, uninstalled launch/config files, C++ breakage on a clean tree | +| `liveliness` | `system/test_liveliness.py` | Containers reach Running, `/clock` publishes, tmux panes alive, sentinel ROS 2 nodes present, compute snapshot, stability poll | Launch files that crash on start, nodes that die after 30 s, `ROBOT_NAME`/domain-ID misconfiguration, runaway CPU or memory | +| `sensors` | `system/test_sensors.py` | Stereo and depth publish rates on both sim and robot side, filtered LiDAR liveness plus geometry sanity, sim real-time factor, time-series stability | Broken sim-to-ROS bridges, sensor Hz that silently halves, RTF collapse from a heavy new node, LiDAR filter range regressions | +| `takeoff_hover_land` | `system/test_takeoff_hover_land.py` | Four-phase chain per (sim, robots, iteration, velocity): PX4 ready → takeoff to 10 m → hover → land | Controller tuning regressions, altitude overshoot, hover drift, state-estimation bias against ground truth, PX4/MAVROS handshake breakage | +| `autonomy` | `system/test_fixed_trajectory.py` | Same chain with a Circle / Figure8 / Racetrack / Line pattern in the middle; records cross-track error and path RMSE | Path-tracker regressions, trajectory-library math errors, velocity/acceleration limit violations that show up as corner-cutting | + +### The flight chain + +Both flight suites run as an ordered chain per parametrization, so the drone +always ends on the ground before the next configuration starts: + +```mermaid +flowchart LR + r["test_px4_ready
MAVROS + EKF"] --> t["test_takeoff
within 10% of 10 m"] + t --> x["test_hover or test_fixed_trajectory"] + x --> l["test_landing
final altitude < 0.5 m"] + r -. "failure" .-> s["remaining phases skipped"] + t -. "failure" .-> s + x -. "failure still lands" .-> l +``` + +A failure in the middle phase (`test_hover` or `test_fixed_trajectory`) does +**not** skip landing — a bad tracker must not leave a drone stuck in the air +blocking the rest of the sweep. A failure in `test_px4_ready` or `test_takeoff` +does skip the remaining phases for that configuration. + +### Bring-up scope, and why mark selection costs money + +`airstack_env` is **class-scoped** and parametrized over +`(sim, num_robots, iteration)`. Each test class does its own `airstack up` and +`airstack down`. Selecting two suites with `or` therefore performs **two full +stack cycles per tuple**: + +```text +-m liveliness → 1 bring-up per (sim, robots, iter) +-m "liveliness or sensors" → 2 bring-ups per (sim, robots, iter) +--sim msairsim,isaacsim → doubles all of the above +--num-robots 1,3 → doubles it again +``` + +Run one mark at a time unless you genuinely need both. + +--- + +## Reading the results + +### The PR comment + +After `run-tests` finishes — pass or fail — a `report` job on `ubuntu-latest` +downloads the current artifact plus a **baseline** artifact and runs +[`parse_metrics.py`](https://github.com/castacks/AirStack/blob/main/tests/parse_metrics.py) +in diff mode. + +| Run type | Baseline used | +|---|---| +| PR opened or `/pytest` | Latest `system-tests.yml` artifact on the PR's base branch | +| `workflow_dispatch` with `baseline_run_id` | That specific run | +| `workflow_dispatch` without it | Latest artifact on `main` | + +The comment has three sections per test module: a flat **Metrics** table, a +**Sim publishing rates** pivot (topic Hz aggregates from the `sensors` mark), +and a **Compute usage** pivot (CPU / memory / GPU per container). Regressions +are marked with a red circle, improvements with a green one, and the job +**fails** if any metric moves more than the 20 % threshold in the wrong +direction. That is the mechanism that catches slow degradation — the kind of +change where nothing throws but the tracker is quietly 30 % worse. + +### The artifact + +`test-results--`, retained 90 days: + +```text +tests/results/2026-08-06_14-30-00/ +├── summary.txt # human-readable per-chain summary — open this first +├── results.xml # JUnit XML: durations, pass/fail per test +└── metrics.json # every recorded metric, including time series +``` + +There are no per-test log files. Live output streams to the Actions log via +pytest's `log_cli`, and failed assertions embed the tail of the relevant +`docker` or `ros2` subprocess output directly in the failure message. + +Regenerate a report locally from a downloaded artifact: + +```bash +python tests/parse_metrics.py \ + --current path/to/current-run/ \ + --baseline path/to/baseline-run/ \ + --threshold 20 +``` + +--- + +## Using CI well while developing + +The pipeline is expensive at the far end and nearly free at the near end. Push +each class of failure as far left as it will go. + +```mermaid +flowchart TD + q{"What did you change?"} + q -- "Pure Python / numpy logic" --> u["airstack test -m unit
seconds, no GPU"] + q -- "Dockerfile / dependency" --> b["airstack test -m build_docker or build_packages
minutes, no GPU"] + q -- "Launch file / new node" --> l["airstack test -m liveliness --sim msairsim --num-robots 1"] + q -- "Sensor or bridge" --> s["airstack test -m sensors --sim isaacsim --num-robots 1"] + q -- "Controller / planner" --> a["airstack test -m autonomy --sim msairsim --trajectory-types Circle"] + u --> pr["Push branch, open PR"] + b --> pr + l --> pr + s --> pr + a --> pr + pr --> ci["Full suite runs on the ephemeral GPU pod"] + ci --> rep["Read the metrics comment"] + rep --> iter["/pytest with a narrowed mark to confirm a fix"] +``` + +Practical rules that follow from how the system is built: + +- **Reproduce CI locally with the same command.** `airstack test` and CI both call `pytest tests/` with the same flags. If a run fails in CI, copy the resolved command from the acknowledgment comment and run it on any GPU box — including an [interactive OSMO dev pod](../../../tutorials/airstack_on_osmo.md) if you do not have a local GPU. +- **Narrow before you re-run.** A `/pytest` with no args re-runs everything. `/pytest -m autonomy --sim msairsim --trajectory-types Circle` re-runs the one chain you are fixing, in a fraction of the time. +- **Never trust a green launch test against a stale build.** This is why `build_packages` is auto-prepended; keep it that way when writing your own `/pytest` line. +- **Read `summary.txt` before the raw log.** It groups each flight chain with per-phase wall times and status, so the failing phase is obvious without scrolling a 40-minute log. +- **Treat the metrics diff as a review artifact.** A PR that turns a metric red needs an explanation in the thread, even when every test passed. +- **Bump `VERSION` in `.env` when image content changes.** [`check-version-increment.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/check-version-increment.yml) gates the PR on a strictly-greater semver, and merging that bump is what triggers the release build below. + +--- + +## The release path + +`system-tests.yml` is not the only workflow on the ephemeral runners. +[`docker-build.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/docker-build.yml) +also requests `runs-on: [self-hosted, airstack-ephemeral]` and therefore gets +the same per-job pod treatment. + +```mermaid +flowchart LR + pr["PR merged to main or develop"] --> chk{".env VERSION changed?"} + chk -- no --> stop["No build"] + chk -- yes --> pod["Ephemeral OSMO pod"] + pod --> plan["Per-service fingerprint plan"] + plan --> retag["imagetools retag unchanged"] + plan --> build["compose build changed only"] + retag --> sign["cosign sign — keyless, GitHub OIDC"] + build --> sign + sign --> verify["cosign verify against the workflow identity"] +``` + +Signing is keyless via GitHub's OIDC token, and the same job immediately +verifies each published digest against the expected certificate identity, so a +published image that was not built by this workflow fails the check. Retagged +images keep the previous digest (and therefore an existing signature still +covers that digest; the job re-signs the same digest under the new tags’ refs). + +| Workflow | Runner | Purpose | +|---|---|---| +| `system-tests.yml` | Ephemeral OSMO GPU pod | Full test suite + metrics report | +| `docker-build.yml` | Ephemeral OSMO GPU pod | Retag or rebuild, push, and sign compose images | +| `check-version-increment.yml` | `ubuntu-latest` | Semver gate on `.env` `VERSION=` | +| `deploy_docs_from_*.yaml` | `ubuntu-latest` | Versioned MkDocs publish via `mike` | + +--- + +## Security model + +| Concern | How the design handles it | +|---|---| +| Cross-job state pollution | Fresh pod per job with an empty Docker cache; destroyed within ~30 s of completion | +| Fork PRs executing arbitrary code on a GPU node | `head.repo.full_name == github.repository` guard on the `pull_request` path, and an explicit fork check plus `author_association` gate on the `/pytest` path | +| Long-lived GitHub PAT on a worker | The PAT lives only on the orchestrator; workers get a single-use JIT config bound to one registration | +| Credentials tied to a person | OSMO auth uses a non-personal service-account token scoped to the CI pool | +| Privileged container is root-equivalent | Accepted deliberately — docker-in-docker is required — but bounded to a one-shot pod, on a dedicated pool, running only same-repo code | +| Orchestrator compromise blast radius | Systemd hardening: `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome=read-only`, `PrivateTmp`, with a single `ReadWritePaths` for state | +| Leaked pods after a crash | Name-prefix orphan sweep plus a `max_job_minutes` straggler ceiling | + +--- + +## Troubleshooting + +A failed run can break at the orchestrator, at the OSMO pod, at the runner, or +in the tests themselves, and each layer has a different inspection path. Work +down the list. + +| Symptom | Layer | First thing to check | +|---|---|---| +| Job sits `queued` forever, no pod appears | Orchestrator | `journalctl -u airstack-orchestrator.service --since '30 min ago'` — look for `submitted workflow for job ` | +| `find_queued_jobs failed: 401` | Orchestrator | GitHub PAT expired or lost a scope; rotate it | +| `osmo login failed` / auth error | Orchestrator | OSMO service-account token expired (default 31 days); mint a new one and restart the service | +| `osmo workflow submit failed ... privileged` | OSMO pool | The pool's platform lacks **Privileged Mode Allowed** | +| Job queued but never claimed | Labels | `runs-on` labels must be a superset of `runner_labels` in `config.yaml` | +| `dockerd did not become ready` | Pod | Not actually privileged; check the platform, then `osmo workflow logs "$WF" --task runner` | +| `nvidia-smi unavailable` | Pod | GPU not requested or the toolkit is not configured on the node | +| `Cannot connect to the Docker daemon` mid-test | Pod | Inner dockerd crashed — `osmo workflow exec "$WF" runner`, then read `/var/log/dockerd.log` | +| `No space left on device` | Pod | Bump `storage` in `config.yaml`; Isaac assets plus all images are large | +| Runner registered, then pytest failed | Tests | A real test failure — the GitHub Actions log and `summary.txt` are canonical | +| Metrics report job failed with no test failures | Report | A metric regressed past the 20 % threshold; read the diff table | + +To map a GitHub job to its pod: + +```bash +JOB_ID=73286176852 # from the GitHub Actions URL +WF=$(sudo jq -r ".jobs[\"$JOB_ID\"].workflow_id" /var/lib/airstack-orchestrator/state.json) + +osmo workflow query "$WF" --verbose +osmo workflow events "$WF" --task runner # scheduling, image pull, eviction +osmo workflow logs "$WF" --task runner # dockerd, run.sh, and the job itself +osmo workflow exec "$WF" runner # break-glass shell, while RUNNING +``` + +Full runbook, including credential rotation and worker-side diagnostics: +[CI/CD Orchestrator](../../../../tests/ci-cd-orchestrator.md). + +--- + +## File map + +| Path | Role | +|---|---| +| [`.github/workflows/system-tests.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/system-tests.yml) | The test workflow: triggers, arg parsing, image prep, pytest, artifact, metrics report | +| [`.github/orchestrator/orchestrator.py`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/orchestrator.py) | The spawn and reap loops, GitHub polling, JIT minting, OSMO CLI plumbing | +| [`.github/orchestrator/runner-workflow.yaml.j2`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-workflow.yaml.j2) | Per-job OSMO workflow template | +| [`.github/orchestrator/runner.Dockerfile`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner.Dockerfile) | Prebaked worker image | +| [`.github/orchestrator/runner-entrypoint.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-entrypoint.sh) | dockerd bring-up, GPU check, single-job runner | +| [`.github/orchestrator/config.example.yaml`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/config.example.yaml) | Every tunable: pool, platform, resources, limits, poll intervals | +| [`.github/orchestrator/setup.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/setup.sh) | One-time orchestrator host install | +| [`tests/conftest.py`](https://github.com/castacks/AirStack/blob/main/tests/conftest.py) | `airstack_env` fixture, collection order, `MetricsRecorder` | +| [`tests/parse_metrics.py`](https://github.com/castacks/AirStack/blob/main/tests/parse_metrics.py) | Report generation and the regression gate | +| [`tests/run_summary.py`](https://github.com/castacks/AirStack/blob/main/tests/run_summary.py) | `summary.txt` generation | + +## See also + +- [System Tests](../../../../tests/README.md) — marks, fixtures, metrics, and every CLI flag. +- [Unit Testing](unit_testing.md) — the co-location and proxy pattern for package-level tests. +- [End-to-End Testing](end_to_end_testing.md) — the fixed-trajectory benchmark in depth. +- [CI/CD Orchestrator](../../../../tests/ci-cd-orchestrator.md) — admin setup, rotation, and break-glass procedures. +- [AirStack on OSMO](../../../tutorials/airstack_on_osmo.md) — interactive GPU dev pods on the same pool. diff --git a/docs/development/intermediate/testing/index.md b/docs/development/intermediate/testing/index.md index e6ce09c0b..087617018 100644 --- a/docs/development/intermediate/testing/index.md +++ b/docs/development/intermediate/testing/index.md @@ -86,4 +86,4 @@ airstack test -m "build_packages or autonomy" \ - [Unit Testing](unit_testing.md) — `@pytest.mark.unit`, proxy pattern, CI workflow - [Testing frameworks](testing_frameworks.md) — `colcon test`, rostest patterns - [Integration testing](integration_testing.md) -- [CI/CD](ci_cd.md) — pipeline overview +- [CI/CD Pipeline on OSMO](ci_cd.md) — how CI runs the full stack on ephemeral GPU pods: architecture, triggers, what each mark catches, and the metrics regression gate diff --git a/docs/tutorials/airstack_on_osmo.md b/docs/tutorials/airstack_on_osmo.md index e9cfa8974..9f093e8c0 100644 --- a/docs/tutorials/airstack_on_osmo.md +++ b/docs/tutorials/airstack_on_osmo.md @@ -547,6 +547,45 @@ osmo workflow cancel $WF +## Batch missions (unattended runs) + +Everything above is the *interactive* workflow. The same pod can instead run +**missions**: declarative YAML files (in +[`osmo/missions/`](https://github.com/castacks/AirStack/blob/main/osmo/missions/)) +that script repeated cycles of bring-up → fly → record → tear-down with no +human attached. Each iteration restarts the containers, records mcap bag +files (Foxglove's native format — open the `.mcap` directly, no conversion), +and snapshots container logs and per-step results. + +```bash +# Submit a mission (auto-pins your current branch, like osmo:up): +./airstack.sh osmo:mission osmo/missions/example_takeoff_land.yaml --pool airstack + +# Watch it fly: +./airstack.sh osmo:logs + +# Pull bags + logs + summaries to your laptop — incremental, safe to run +# mid-mission and again later to top up: +./airstack.sh osmo:fetch ./results/ + +# When you have everything (results die with the pod!): +./airstack.sh osmo:down +``` + +A mission step can be any robot task action (`takeoff`, `land`, `navigate`, +`semantic_search`, `exploration`, `coverage`, …), a timed wait, a topic pub, +a service call, or an arbitrary `ros2`/shell command. Spec schema and step +reference: +[`osmo/missions/README.md`](https://github.com/castacks/AirStack/blob/main/osmo/missions/README.md). + +By default the pod **stays alive after the mission ends** so you can +`osmo:fetch` whenever you're ready (mind the workflow's 24h `exec_timeout`). +For fire-and-forget batches, submit with `--no-keep-alive`: the pod exits +cleanly when the mission ends, freeing the GPU — and uploading the results +directory to object storage automatically if the workflow's `outputs:` +block is configured (lab-admin setup; see +[`osmo/README.md`](https://github.com/castacks/AirStack/blob/main/osmo/README.md)). + ## Troubleshooting | Symptom | Likely cause | Fix | @@ -572,7 +611,8 @@ osmo workflow cancel $WF | Uncommitted edits in the IDE | Pod-local working tree | **No** | | `colcon build` outputs (`build/`, `install/`, `log/`) | `/root/AirStack/**/ros_ws/...` | **No** (gitignored Linux x86_64 binaries; rebuild trivially) | | Inner-dockerd image cache | Pod-local Docker layer cache | **No** | -| Bag files, sim recordings, debug screenshots | `/root/AirStack/bags/`, etc. | **No** — pull selectively via `osmo workflow rsync download "$(cat ~/.airstack/osmo-state)" :` *before* tearing down | +| Mission results (mcap bags, logs, summaries) | `/root/AirStack/osmo/results/` | **No** — run `./airstack.sh osmo:fetch` *before* tearing down | +| Other bag files, sim recordings, debug screenshots | `/root/AirStack/bags/`, etc. | **No** — pull selectively via `osmo workflow rsync download "$(cat ~/.airstack/osmo-state)" :` *before* tearing down | The rule of thumb: **commit + push every time you'd save a file in a git-tracked sense.** The Source Control panel is the persistence boundary. diff --git a/gcs/docker/gcs-base-docker-compose.yaml b/gcs/docker/gcs-base-docker-compose.yaml index c8ac5200e..8894bcb9f 100644 --- a/gcs/docker/gcs-base-docker-compose.yaml +++ b/gcs/docker/gcs-base-docker-compose.yaml @@ -7,8 +7,10 @@ services: dockerfile: docker/Dockerfile.gcs tags: - &gcs_image ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:v${VERSION}_gcs + - &gcs_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_gcs cache_from: - *gcs_image + - *gcs_cache command: > bash -c " ssh service restart; diff --git a/mkdocs.yml b/mkdocs.yml index e75d85bc7..c4845657e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,6 +70,7 @@ nav: - Unit Testing: docs/development/intermediate/testing/unit_testing.md - System Tests: tests/README.md - End-to-End Testing: docs/development/intermediate/testing/end_to_end_testing.md + - CI/CD Pipeline: docs/development/intermediate/testing/ci_cd.md - CI/CD Orchestrator: tests/ci-cd-orchestrator.md - Frame Conventions: docs/development/intermediate/frame_conventions.md - Docker Build Profiles: docs/development/intermediate/docker-build-profiles.md diff --git a/osmo/README.md b/osmo/README.md index 91b41dbd5..410ed21c2 100644 --- a/osmo/README.md +++ b/osmo/README.md @@ -7,11 +7,17 @@ through [NVIDIA OSMO](https://github.com/NVIDIA/OSMO): osmo/ ├── README.md # This file (admin / operator reference) ├── workflows/ -│ └── airstack-dev.yaml # The OSMO workflow students submit +│ ├── airstack-dev.yaml # Interactive dev workflow (IDE over Remote-SSH) +│ └── airstack-mission.yaml # Batch mission workflow (unattended flights) +├── missions/ +│ ├── README.md # Mission spec schema reference +│ └── example_takeoff_land.yaml # Reference mission: takeoff → hover → land ×3 └── workspace/ ├── Dockerfile # The airstack-osmo-workspace image ├── sshd_config # Pubkey-only sshd config baked into the image - └── entrypoint.sh # Pod startup: sshd, dockerd, clone, airstack up + ├── entrypoint.sh # Pod startup: sshd, dockerd, clone, then + │ # dev mode (airstack up) or mission mode + └── mission_runner.py # Batch executor (run from the clone, not the image) ``` The student-facing walkthrough lives in @@ -21,9 +27,9 @@ README is the **lab admin / operator** reference: pool requirements, workspace image build & push, validation stages, plus a credential summary for context. -> **Scope:** developer workflow only. CI/CD on OSMO is **not** part of this -> integration — the existing `system-tests.yml` + OpenStack orchestrator path -> is unchanged. +> **Scope:** developer workflow + batch missions. CI/CD on OSMO is **not** +> part of this integration — the existing `system-tests.yml` + OpenStack +> orchestrator path is unchanged. ## Architecture in one minute @@ -47,6 +53,49 @@ app.foxglove.dev ── ws ────► port-forward 8766 ────► airstack.sh up brings these 3 up ``` +## Mission mode (batch runs) + +`airstack-mission.yaml` reuses the same workspace image and DinD pod, but +instead of one interactive `airstack up`, the entrypoint hands off to +[`workspace/mission_runner.py`](workspace/mission_runner.py), which executes +a declarative mission spec from [`missions/`](missions/) — repeated cycles of: + +``` +airstack down → airstack up → wait for PX4 ready → record mcap bags +→ run steps (takeoff / land / navigate / semantic search / any ros2 command) +→ collect bags + container logs → airstack down +``` + +Submit, monitor, and download: + +```bash +airstack osmo:mission osmo/missions/example_takeoff_land.yaml --pool +airstack osmo:logs # follow mission progress +airstack osmo:fetch ./results/ # rsync bags/logs/summaries to the laptop +airstack osmo:down # cancel (fetch first — results die with the pod) +``` + +Key behaviors: + +- **The mission spec and runner come from the clone**, not the image — what + you push on your branch is what runs. The workspace image only needs a + rebuild when `Dockerfile`, `sshd_config`, or `entrypoint.sh` change. +- **Bags are mcap** (`ros2 bag record -s mcap`) — open the `.mcap` files + directly in Foxglove, no conversion or local ROS install. +- **Results location:** `/osmo/output/airstack-mission-results///` + with a symlink at `/root/AirStack/osmo/results` (the path `osmo:fetch` + pulls). Artifacts are collected even for failed iterations. +- **`OSMO_MISSION_KEEP_ALIVE`** (default `true`): the pod sleeps after the + mission so you can fetch over ssh. Set `false` (or submit with + `osmo:mission --no-keep-alive`) for fire-and-forget: the task exits + cleanly when the mission ends, freeing the GPU — and if the workflow's + `outputs:` block is configured with a destination bucket, OSMO uploads + `/osmo/output` automatically on that exit. A **canceled** workflow does + not upload outputs, so in keep-alive mode `osmo:fetch` is the retrieval + path. + +Mission spec schema and step types: [`missions/README.md`](missions/README.md). + ## Pool requirements The OSMO pool the workflow runs on must satisfy: diff --git a/osmo/missions/README.md b/osmo/missions/README.md new file mode 100644 index 000000000..f33b977e7 --- /dev/null +++ b/osmo/missions/README.md @@ -0,0 +1,182 @@ +# AirStack mission specs + +A **mission** is a declarative YAML file executed by +[`osmo/workspace/mission_runner.py`](../workspace/mission_runner.py). Each +mission runs one or more full iterations of: + +``` +airstack down → airstack up → wait for PX4 ready → start mcap recording +→ run steps → stop recording → collect bags + container logs → airstack down +``` + +Missions live in this directory so they're versioned with the code they +exercise — the OSMO pod clones your branch, so whatever you push is what runs. + +Submit to OSMO with: + +```bash +airstack osmo:mission osmo/missions/.yaml --pool +``` + +Or run locally on any machine that can `airstack up` (no OSMO involved): + +```bash +python3 osmo/workspace/mission_runner.py osmo/missions/.yaml \ + --airstack-root "$(pwd)" +``` + +`--dry-run` validates the spec and prints the merged config without touching +Docker. + +## Results + +``` +osmo/results/// +├── summary.json # per-iteration status + durations +└── iter_001/ + ├── bags/robot_1/*.mcap # open directly in Foxglove (no conversion) + ├── logs/.log # docker logs snapshot per container + ├── logs// # raw /tmp stdout tees (gossip, ddsrouter, relay) + rcl control-stack logs + ├── ready.json # per-robot seconds-to-PX4-ready + ├── steps.json # per-step command, output tail, pass/fail + └── iteration.json # iteration summary +``` + +On an OSMO pod the actual storage is `/osmo/output/airstack-mission-results` +(with `osmo/results` symlinked to it), so a workflow `outputs:` block uploads +everything automatically when the task exits. Download from your laptop at +any time while the pod is alive with `airstack osmo:fetch [dest]`. + +Artifacts are collected **even when an iteration fails** — a failed flight's +bag is usually the most interesting one. + +## Schema + +Top-level keys (everything except `steps` is optional): + +| Key | Default | Meaning | +|---|---|---| +| `name` | filename stem | Results directory name | +| `env` | `{}` | Env vars exported before each `airstack up` (`NUM_ROBOTS`, `COMPOSE_PROFILES`, `ISAAC_SIM_SCRIPT_NAME`, …) | +| `iterations` | `1` | Full up→fly→down cycles. With `environments:` + `environment_order: grouped`, this is **per environment** (total = iterations × #environments) | +| `iteration_attempts` | `1` | Max times to (re)run a single iteration until it passes. `>1`: a failed/errored iteration is redone (clean down→up→fly→down) up to this many attempts; a `passed`, manually `stopped`, or `abort_mission` outcome is never retried. Failed attempts' artifacts are preserved under `iter_NNN_failed_attempt_K`. With a fixed `SPAWN_SEED` the redo reproduces the same spawn layout | +| `environment_order` | `round_robin` | With `environments:`: `round_robin` (iteration i → env[(i-1) % n], `iterations` is the total) \| `grouped` (each env runs `iterations` times in a row) | +| `ready.timeout_s` | `600` | Max seconds to wait for PX4 readiness per iteration | +| `ready.poll_interval_s` | `5` | Seconds between readiness polls | +| `record.enabled` | `true` | Record an mcap per robot per iteration | +| `record.scope` | `gcs` | `gcs` (one mcap on GCS domain 0) \| `robot` (one mcap per robot domain) \| `both` | +| `record.topics` | tf + odom set | Topics to record; `{robot}` → `robot_N` | +| `record.all` | `false` | Record **all** topics (`ros2 bag record -a`) — large | +| `record.exclude` | — | With `all`: regex of topics to drop (`ros2 bag record -a --exclude-regex `) | +| `record.required` | `false` | Abort the iteration if any recorder fails to start (don't fly unrecorded) | +| `on_step_failure` | `abort_iteration` | `continue` \| `abort_iteration` \| `abort_mission` | +| `up_timeout_s` | `3600` | `airstack up` timeout (first up on a fresh pod pulls images) | +| `down_timeout_s` | `300` | `airstack down` timeout | +| `robot_setup_bash` | robot ws `setup.bash` | Workspace sourced before `ros2` commands | +| `nas_dest` | — | Base path on airlab-storage (e.g. `/volume3//airstack-missions`). When the `airlab-storage` OSMO credential is set (`airstack osmo:setup`), an OSMO pod rsyncs results to `///` then tears itself down. Override per run with `osmo:mission --nas-dest PATH`; suppress with `--no-nas-upload`. Credentials never live in the spec | +| `steps` | — | Ordered list of steps (below) | + +### Steps + +Every step type accepts the placeholders `{robot}` → `robot_N` and `{n}` → `N` +in its strings, and runs once per robot in `robots:` (`all` by default, or a +list like `[1, 3]`). + +**`action`** — send a goal to a robot task action server and wait for the +result. The step passes when the action result reports `success: true`. + +```yaml +- action: + task: takeoff # → /robot_N/tasks/ + goal: {target_altitude_m: 10.0, velocity_m_s: 1.0} + timeout_s: 120 # default 120 (per attempt) + attempts: 3 # per-robot retries on failure (default 3) + retry_delay_s: 10 # wait between attempts (default 10) + feedback_timeout_s: 15 # via gcs: no relay_feedback within this + # window ⇒ goal presumed lost, retried + pass_on_feedback: false # via gcs: pass as soon as feedback is seen + # (the task RAN), regardless of success/fail; + # only "never ran" (no feedback + no result + # after retries) fails the step + robots: all + # type: task_msgs/action/TakeoffTask # derived from task name if omitted +``` + +Any step may also carry **`optional: true`** (a sibling key, not inside +`action`/`run`/…): the step still runs and its result is recorded, but a +failure neither trips `on_step_failure` nor counts toward an `iteration_attempts` +redo. Use it for steps whose outcome doesn't gate the iteration — e.g. a `land` +after the run is already done. + +**`pass_on_feedback` + `iteration_attempts` idiom** — to guarantee a task +*runs* every iteration without caring whether it succeeds: set +`pass_on_feedback: true` on that action and `iteration_attempts: >1` on the +mission. If the task never gets feedback (goal lost / never started) after its +`attempts` retries, the step fails and the whole iteration is redone; once +feedback is seen the iteration is considered satisfied. + +Each robot's goal is logged per attempt and retried independently — a goal +can be rejected transiently (relay has no GPS fix yet, PX4 position estimate +not converged, action server still starting), so one robot failing its first +attempt doesn't fail the step unless it exhausts all attempts. + +Available tasks (action type is derived as `task_msgs/action/Task`): +`takeoff`, `land`, `fixed_trajectory`, `navigate`, `exploration`, `coverage`, +`semantic_search`, `chat`. Goal fields are defined in +[`common/ros_packages/msgs/task_msgs/action/`](../../common/ros_packages/msgs/task_msgs/action/). +Multi-robot action goals are sent **in parallel** across robots. + +**`wait`** — sleep for N seconds (e.g. hover, let a planner run): + +```yaml +- wait: 30 +``` + +**`run`** — arbitrary command; the escape hatch that makes any ROS 2 command +work without runner changes. The step fails on non-zero exit unless +`expect_success: false`. Set `attempts` to retry on failure (e.g. a flaky +model download) before the step is marked failed. + +```yaml +- run: + container: robot_1 # robot_N → exec in the robot container on robot + # N's DDS domain (ros2 is sourced for you); + # pod → run on the pod itself (cwd = AirStack root); + # any other value → literal container name, domain 0 + cmd: ros2 topic echo --once /{robot}/odometry + timeout_s: 60 + expect_success: true + attempts: 1 # retries on failure (default 1 = no retry) + retry_delay_s: 10 # wait between attempts (default 10) +``` + +**`topic_pub`** — `ros2 topic pub --once` per robot: + +```yaml +- topic_pub: + topic: /{robot}/some_input + type: std_msgs/msg/Bool + msg: {data: true} +``` + +**`service_call`** — `ros2 service call` per robot: + +```yaml +- service_call: + service: /{robot}/some_service + type: std_srvs/srv/Trigger + request: {} +``` + +## Notes + +- For `NUM_ROBOTS > 1` on Isaac Sim, set + `ISAAC_SIM_SCRIPT_NAME: example_multi_px4_pegasus_launch_script.py` in + `env:` — the default script spawns a single drone. +- Missions run unattended: keep `ISAAC_SIM_HEADLESS: "true"`, + `ISAAC_SIM_USE_STANDALONE: "true"` and `PLAY_SIM_ON_START: "true"` unless + you're watching via the WebRTC livestream profile. +- Recording camera/LiDAR topics (or `record.all: true`) is the bag-size + driver — budget pod `storage:` accordingly. +- `/tf` and `/tf_static` are in the default topic set because without them a + Foxglove 3D panel can't pose anything during replay. diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml new file mode 100644 index 000000000..237447b12 --- /dev/null +++ b/osmo/missions/example_takeoff_land.yaml @@ -0,0 +1,159 @@ +# Example mission: takeoff → hover 30s → land, cycled across two Isaac Sim +# scenes (headless), one iteration per scene. +# +# Run on OSMO: +# airstack osmo:mission osmo/missions/example_takeoff_land.yaml --pool +# +# Or locally against a machine that can run `airstack up` (no OSMO needed): +# python3 osmo/workspace/mission_runner.py osmo/missions/example_takeoff_land.yaml \ +# --airstack-root "$(pwd)" +# +# Schema reference: osmo/missions/README.md + +name: example_takeoff_land + +# Optional: upload results to airlab-storage, then auto-teardown once the +# upload succeeds (needs the airlab-storage OSMO credential — see +# `airstack osmo:setup`). Override the path with osmo:mission --nas-dest, +# or suppress with --no-nas-upload. +# nas_dest: /volume//airstack-missions + +# How to bring the stack up. `services` are passed straight through: +# ./airstack.sh up isaac-sim robot-desktop gcs +# (docker compose auto-enables a named service's profile, so no +# COMPOSE_PROFILES needed). On OSMO swap isaac-sim → isaac-sim-livestream to +# watch the run over WebRTC. Alternatively use `profiles:` to export +# COMPOSE_PROFILES and run a plain `airstack up`. +stack: + services: [isaac-sim-livestream, robot-desktop, gcs] + # profiles: [desktop, isaac-sim] + +# Exported before every `airstack up`. Anything the compose stack reads from +# the environment can go here (.env values, ISAAC_SIM_SCRIPT_NAME, etc.). +env: + NUM_ROBOTS: 1 + ISAAC_SIM_HEADLESS: "true" + ISAAC_SIM_USE_STANDALONE: "true" + # Reads the per-environment overrides (ENV_URL, STAGE_SCALE, SPAWN_CONFIGS, + # SPAWN_HEIGHT_M) exported from `environments:` below. Without environments, + # use example_one_px4_pegasus_launch_script.py for a single default drone. + ISAAC_SIM_SCRIPT_NAME: example_multi_drone_scene_import.py + # Missions are unattended — the sim must start playing without a GUI click. + PLAY_SIM_ON_START: "true" + +# Environments cycled across iterations (optional — drop this list to run +# every iteration in the launch script's default scene). +# +# UPPERCASE keys are exported as env vars before `airstack up`, so the +# isaac-sim launch script picks them up: +# ENV_URL: full omniverse://… URL, or just the path after the nucleus +# server (the launch script prepends omniverse:///) +# STAGE_SCALE: stage meters-per-unit scale factor +# SPAWN_CONFIGS: fixed per-drone spawn list; only x_m / y_m required +# (z_m, orient, lidar_min_range default per drone) +# SPAWN_HEIGHT_M: default spawn height above the floor +# +# lowercase keys are per-environment step arguments, referenced from any step +# via {{env.}} — e.g. takeoff_altitude_m below sets a different takeoff +# height per scene. +environments: + - name: fireacademy + ENV_URL: "Projects/AirStack/scenes/urban/allegheny_county_fire_academy/fire_academy.scene.usd" + STAGE_SCALE: "0.01" + SPAWN_CONFIGS: + - {x_m: 32.0, y_m: 12.6, orient: [0.0, 0.0, -0.937, 0.35]} + SPAWN_HEIGHT_M: "0.03" + takeoff_altitude_m: 10.0 + + - name: retroneighborhood + ENV_URL: "Library/Stages/RetroNeighborhood/RetroNeighborhood.stage.usd" + STAGE_SCALE: "0.01" + SPAWN_CONFIGS: + - {x_m: 6.764, y_m: -1.767, orient: [0.0, 0.0, -0.1193, 0.9929]} + SPAWN_HEIGHT_M: "0.3" + # Taller buildings — take off higher in this scene. + takeoff_altitude_m: 20.0 + +# round_robin(default): iteration i runs environments[(i-1) % n], so +# `iterations` is the TOTAL count — 2 here = one run per scene. +# grouped: each environment runs `iterations` times in a row. +environment_order: grouped + +# Where `action` steps are sent by default: +# gcs — through the GCS action_relay, the same String-JSON → typed-goal +# path Foxglove and the GCS panels use (exercises the full +# GCS→robot chain, including the relay's airborne preconditions) +# robot — `ros2 action send_goal` directly on the robot's DDS domain +# Override per step with `via:`. +command_route: gcs + +# Full up → fly → collect → down cycles (total across environments — see +# environment_order above). +iterations: 2 + +# Gate before any steps run: per robot, MAVROS connected, then +# local_position/odom publishing (PX4 EKF converged = ready to arm). +ready: + timeout_s: 600 + +# One mcap per iteration with ALL robots' data, recorded on the GCS +# (domain 0) where each robot's domain_bridge forwards its state topics — +# replaying the .mcap in Foxglove reproduces the live GCS view. +# {robot} expands to robot_1..robot_N and the list is unioned. +record: + enabled: true + scope: gcs # or `robot`: one mcap per robot on its own + # domain, for topics not bridged to the GCS + topics: + - /tf + - /tf_static + - /{robot}/odometry_conversion/odometry + - /{robot}/interface/mavros/global_position/global + - /{robot}/trajectory_controller/trajectory_vis + - /{robot}/global_plan + - /gcs/robot_markers + - /gcs/map_origin/location + - /gcs/map_origin/ground_msl + - /gcs/{robot}/location + # Or record everything visible on the recording domain (large): + # all: true + +# continue | abort_iteration (default) | abort_mission +on_step_failure: abort_iteration + +# Steps run top to bottom. Every step that addresses a robot fans out over a +# robot selection — `robots:` (default `all` = robots 1..NUM_ROBOTS, or a +# list like [1, 3]) — and {robot} → robot_N / {n} → N expand per robot. +# Per-step knobs (all optional, shown here with their defaults): +# action: robots=all via= timeout_s=120 (per attempt) +# attempts=3 retry_delay_s=10 (per-robot retries on failure) +# via: gcs (through the GCS action_relay) | robot (direct send_goal) +# run: robots=all container=robot_{n} (per-robot) or robot_1 | gcs | pod | +# ; fans out when cmd/container references {robot}/{n} +# wait: (runs once) +steps: + - action: + task: takeoff # → //tasks/takeoff (TakeoffTask) + # {{env.takeoff_altitude_m}} resolves per iteration from the current + # `environments:` entry above. + goal: {target_altitude_m: "{{env.takeoff_altitude_m}}", velocity_m_s: 1.0} + # robot selection (default all): + #robots: [1] + #robots: [1, 3] + #robots: all + + - wait: 30 # hover + + - run: # arbitrary command, that runs per robot + cmd: ros2 topic echo --once /{robot}/odometry_conversion/odometry + timeout_s: 20 + # robot selection (default all): + #robots: [1] + #robots: [1, 3] + # container to run in (default robot_{n}): + #container: gcs + #container: + + - action: + task: land # → //tasks/land (LandTask) + goal: {velocity_m_s: 0.5} diff --git a/osmo/workflows/airstack-mission.yaml b/osmo/workflows/airstack-mission.yaml new file mode 100644 index 000000000..7429cae27 --- /dev/null +++ b/osmo/workflows/airstack-mission.yaml @@ -0,0 +1,138 @@ +# AirStack batch mission workflow on OSMO. +# +# Same workspace image and pod layout as airstack-dev.yaml, but instead of a +# single `airstack up` for an interactive session, this runs a batch mission: +# osmo/workspace/mission_runner.py executes the mission spec named by +# OSMO_MISSION_FILE — repeated cycles of +# +# airstack down → airstack up → wait for PX4 ready → record mcap bags +# → run steps (takeoff / land / navigate / any ros2 command) → collect +# bags + logs → airstack down +# +# NO IMAGE REBUILD NEEDED. The mission engine is NOT baked into the image: +# the command below reuses the baked entrypoint for pod setup only +# (OSMO_AIRSTACK_UP=false → sshd, inner dockerd, branch clone, creds, registry +# login — no `airstack up`), then hands off to mission_launcher.sh FROM THE +# CLONE. So `git push` your branch and resubmit; the launcher + runner + spec +# all come from the branch. (The launcher pip-installs PyYAML at runtime if +# the image predates it.) +# +# Mission specs live in osmo/missions/ (cloned with the branch, so what you +# push is what runs). Schema: osmo/missions/README.md. +# +# To submit (or use the wrapper: airstack osmo:mission ): +# +# osmo workflow submit osmo/workflows/airstack-mission.yaml \ +# --pool \ +# --set-env "SSH_PUB_KEY=$(cat ~/.ssh/id_ed25519.pub)" \ +# "OSMO_MISSION_FILE=osmo/missions/example_takeoff_land.yaml" \ +# "AIRSTACK_BRANCH=" +# +# Results land in /osmo/output/airstack-mission-results (symlinked from +# /root/AirStack/osmo/results). With the default OSMO_MISSION_KEEP_ALIVE=true +# the pod stays alive after the mission so you can pull artifacts from your +# laptop: +# +# airstack osmo:fetch ./results/ +# +# Set OSMO_MISSION_KEEP_ALIVE=false for fire-and-forget runs: the task exits +# cleanly when the mission ends, which frees the GPU and triggers the +# `outputs:` upload below (if a destination bucket is configured). +# +# NAS upload: if the airlab-storage credential is set (airstack osmo:setup) and +# the mission spec has `nas_dest:`, the pod rsyncs results to airlab-storage +# then tears itself down — no laptop fetch needed. See mission_launcher.sh. + +workflow: + name: airstack-mission + groups: + - name: airstack + tasks: + - name: workspace + lead: true + image: airlab-docker.andrew.cmu.edu/airstack/airstack-osmo-workspace:latest + # Required so the inner dockerd can run (same DinD setup as airstack-dev). + privileged: true + # Bootstrap: run the baked entrypoint for pod setup ONLY + # (OSMO_AIRSTACK_UP=false), backgrounded, then hand off to the launcher + # that ships in the freshly-cloned branch. This is what makes the + # mission engine rebuild-free — see the header comment. + command: ["bash", "-c"] + args: + - | + set -uo pipefail + log() { echo "[mission-bootstrap] $*"; } + log "starting pod setup via baked entrypoint (OSMO_AIRSTACK_UP=false)" + # Baked entrypoint for pod setup, backgrounded. `tee` streams its logs to + # `osmo workflow logs` and to a file the launcher greps for the terminal + # "sleeping forever" line. `tee` holds THIS task's stdout pipe open, so + # mission_launcher.sh kills BAKED_PID before it exits — otherwise + # osmo_exec never sees pipe EOF on our exit and the pod never frees the + # GPU (it hangs at RUNNING until exec_timeout). + OSMO_AIRSTACK_UP=false /usr/local/bin/entrypoint.sh 2>&1 \ + | tee /tmp/baked-entrypoint.log & + export BAKED_PID=$! + LAUNCHER=/root/AirStack/osmo/workspace/mission_launcher.sh + log "waiting for branch clone to provide $LAUNCHER" + for i in $(seq 1 600); do [ -f "$LAUNCHER" ] && break; sleep 2; done + if [ ! -f "$LAUNCHER" ]; then + log "ERROR: $LAUNCHER not found after clone wait; sleeping for SSH debug" + exec sleep infinity + fi + log "handing off to mission_launcher.sh" + exec bash "$LAUNCHER" + environment: + # Mission selection — repo-relative path into the clone. Override at + # submit time to run a different mission. + OSMO_MISSION_FILE: "osmo/missions/example_takeoff_land.yaml" + # true → pod sleeps after the mission; fetch results over ssh, then + # `airstack osmo:down` (results die with the pod!). + # false → task exits when the mission ends: GPU freed, /osmo/output + # uploaded to `outputs:` destinations. + OSMO_MISSION_KEEP_ALIVE: "true" + AIRSTACK_BRANCH: "krrishj18/osmo-mission-runner" # branch entrypoint.sh clones + AIRSTACK_REPO_URL: "https://github.com/castacks/AirStack.git" + # SSH_PUB_KEY is supplied at submit time (needed for osmo:fetch): + # --set-env "SSH_PUB_KEY=$(cat ~/.ssh/id_ed25519.pub)" + # Uncomment and point at a bucket the OSMO deployment can write to, to + # have /osmo/output uploaded automatically when the task exits (only + # meaningful with OSMO_MISSION_KEEP_ALIVE=false — a canceled workflow + # does not upload outputs): + # + # outputs: + # - url: s3:///airstack-missions/{{workflow_id}}/ + credentials: + # Same per-user credentials as airstack-dev.yaml — see + # docs/tutorials/airstack_on_osmo.md "Step 0" / `airstack osmo:setup`. + # These map OSMO credential fields → pod env vars; no secret values live + # in this file. + airlab-nucleus: + OMNI_USER: omni_user + OMNI_PASS: omni_pass + OMNI_SERVER: omni_server + airlab-docker-login: + AIRLAB_REGISTRY_USER: username + AIRLAB_REGISTRY_PASS: password + # Optional: NAS result upload. Absent credential → nothing injected → + # upload disabled. Enabled per-mission via `nas_dest:` in the spec. + airlab-storage: + AIRLAB_STORAGE_USER: username + AIRLAB_STORAGE_PASS: password + AIRLAB_STORAGE_HOST: host + + resources: + default: + cpu: 16 + gpu: 1 + memory: 64Gi + # Same sizing rationale as airstack-dev.yaml (inner image set alone + # exceeds 100Gi extracted), plus headroom for per-iteration mcap + # recordings — bump if a mission records camera/LiDAR over many + # iterations. + storage: 500Gi + + timeout: + # With keep-alive the pod sleeps after the mission until this expires (then + # it's gone — fetch first). Sized to stay up ~30h post-mission for + # osmo:fetch (mission itself runs in the first ~1-2h). + exec_timeout: 32h diff --git a/osmo/workspace/Dockerfile b/osmo/workspace/Dockerfile index e80f3be59..d6a43f5d1 100644 --- a/osmo/workspace/Dockerfile +++ b/osmo/workspace/Dockerfile @@ -42,7 +42,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ procps \ python3 \ python3-pip \ + python3-yaml \ rsync \ + sshpass \ sudo \ tmux \ tzdata \ diff --git a/osmo/workspace/entrypoint.sh b/osmo/workspace/entrypoint.sh index bef9d9ba8..dd53a94ae 100755 --- a/osmo/workspace/entrypoint.sh +++ b/osmo/workspace/entrypoint.sh @@ -259,12 +259,17 @@ else export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim}" fi +# OSMO_AIRSTACK_UP=false skips `airstack up` and goes straight to sleep — the +# pod is fully set up (sshd, dockerd, clone, creds) for the caller to drive. +# The airstack-mission.yaml workflow uses this hook: it runs this entrypoint +# for setup only, then hands off to osmo/workspace/mission_launcher.sh (from +# the clone, so the mission engine ships with the branch — no image rebuild). cd "$AIRSTACK_ROOT" if [ "${OSMO_AIRSTACK_UP:-true}" = "true" ]; then log "airstack up (COMPOSE_PROFILES=$COMPOSE_PROFILES, NUM_ROBOTS=$NUM_ROBOTS, livestream=$ISAAC_SIM_LIVESTREAM)" ./airstack.sh up || log "WARN: airstack up exited non-zero — pod stays alive for debugging via SSH" else - log "OSMO_AIRSTACK_UP=false — skipping airstack up; SSH in and run ./airstack.sh up manually" + log "OSMO_AIRSTACK_UP=false — skipping airstack up; caller drives the stack" fi # ─── 7. Sleep ────────────────────────────────────────────────────────────── diff --git a/osmo/workspace/mission_launcher.sh b/osmo/workspace/mission_launcher.sh new file mode 100755 index 000000000..b470d10c7 --- /dev/null +++ b/osmo/workspace/mission_launcher.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# mission_launcher.sh — no-rebuild mission entrypoint for AirStack-on-OSMO. +# +# The airstack-mission.yaml workflow runs the *baked* image entrypoint with +# OSMO_AIRSTACK_UP=false (pod setup only: sshd, inner dockerd, branch clone, +# Nucleus creds, registry login), then hands off to THIS script — which lives +# in the clone, so the mission engine always comes from your branch and no +# workspace-image rebuild is needed to iterate. (PyYAML, which the baked image +# may predate, is installed at runtime below.) +# +# Responsibilities: +# 1. Wait for the setup the baked entrypoint performs in the background +# (inner dockerd ready, clone present, registry login done). +# 2. Ensure PyYAML is importable. +# 3. Run mission_runner.py against $OSMO_MISSION_FILE. +# 4. Keep-alive: stay foreground (sleep) so the pod survives for +# `airstack osmo:fetch`, or exit so OSMO uploads /osmo/output + frees GPU. + +set -uo pipefail + +AIRSTACK_ROOT="${AIRSTACK_ROOT:-/root/AirStack}" + +log() { echo "[mission-launcher] $*"; } +fail() { echo "[mission-launcher] ERROR: $*" >&2; } + +# Kill the backgrounded `tee` (BAKED_PID, from the workflow bootstrap) so it +# releases this task's OSMO stdout pipe — else osmo_exec never sees EOF on our +# exit and the pod never frees the GPU. Call right before an intentional exit. +release_osmo_pipe() { [ -n "${BAKED_PID:-}" ] && kill "$BAKED_PID" 2>/dev/null; true; } + +# Fast DDS LARGE_DATA uses shared memory between same-host/same-version +# participants; the segments live in /dev/shm, which every stack container +# bind-mounts from the pod. The pod default is 64M — it fills immediately +# and later participants then fail SHM registration. Regrow in place. +mount -o remount,size=8G /dev/shm 2>/dev/null +log "/dev/shm: $(df -h /dev/shm | awk 'NR==2{print $2" total, "$5" used"}')" + +# wait_for — poll until the command +# succeeds or the timeout elapses. Returns non-zero on timeout. +wait_for() { + local desc="$1" timeout="$2"; shift 2 + local i=0 + until "$@" >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -ge "$timeout" ]; then + fail "timed out after ${timeout}s waiting for: ${desc}" + return 1 + fi + [ $((i % 15)) -eq 0 ] && log "still waiting for ${desc} (${i}s)" + sleep 1 + done + log "ready: ${desc}" +} + +# ── 1. setup readiness (driven by the backgrounded baked entrypoint) ─────── +# If dockerd or the clone never appear, the pod is broken — stay alive (when +# keep-alive) so it can be inspected over SSH rather than vanishing. +if ! wait_for "inner dockerd" 180 docker info; then + [ "${OSMO_MISSION_KEEP_ALIVE:-true}" = "true" ] && exec sleep infinity + exit 1 +fi +if ! wait_for "branch clone" 600 test -f "$AIRSTACK_ROOT/osmo/workspace/mission_runner.py"; then + [ "${OSMO_MISSION_KEEP_ALIVE:-true}" = "true" ] && exec sleep infinity + exit 1 +fi + +# The deployed :latest image's baked entrypoint ignores OSMO_AIRSTACK_UP and +# runs its OWN `airstack up`. Running the mission's bring-up concurrently +# collides (duplicate network, container-name conflicts, "network not found" +# mid-teardown) and fails the first iteration. So wait for the baked +# entrypoint to FINISH — the bootstrap tees its output to $BAKED_LOG, and its +# terminal "sleeping forever" line is printed exactly once, after its +# `airstack up` completes (success or fail). This also subsumes the clone + +# registry-login steps it performs, and warms the inner image cache the +# mission reuses. On a rebuilt image where the hook works, the entrypoint +# skips `up` and prints that line almost immediately, so this is a fast no-op. +# +# We deliberately do NOT detect completion via `pgrep -f "sleep infinity"`: +# the AirStack containers themselves run `sleep infinity`, so that matches the +# instant the baked bring-up starts a container — firing the handoff in the +# middle of its `airstack up`, which is exactly the race we're avoiding. +BAKED_LOG="${BAKED_ENTRYPOINT_LOG:-/tmp/baked-entrypoint.log}" +log "waiting for the baked entrypoint to finish its own bring-up (avoids a concurrent 'airstack up')" +if ! wait_for "baked entrypoint complete" 2400 grep -q "sleeping forever" "$BAKED_LOG"; then + log "WARN: baked entrypoint didn't signal completion in 40m — proceeding; first iteration may race its bring-up" +fi +sleep 5 # let baked's compose settle before the mission's first ensure_down/up + +# ── 2. PyYAML (mission_runner imports yaml) ──────────────────────────────── +if ! python3 -c "import yaml" >/dev/null 2>&1; then + log "installing PyYAML (image predates python3-yaml)" + pip3 install --break-system-packages --quiet pyyaml \ + || { apt-get update -qq && apt-get install -y -qq python3-yaml; } \ + || fail "could not install PyYAML — mission_runner will fail to import yaml" +fi + +# ── 3. run the mission ───────────────────────────────────────────────────── +MISSION="${OSMO_MISSION_FILE:-}" +if [ -z "$MISSION" ]; then + fail "OSMO_MISSION_FILE not set — nothing to run" +else + MISSION_PATH="$AIRSTACK_ROOT/$MISSION" + [ -f "$MISSION_PATH" ] || MISSION_PATH="$MISSION" # allow an absolute path + if [ -f "$MISSION_PATH" ]; then + log "running mission: $MISSION_PATH" + python3 "$AIRSTACK_ROOT/osmo/workspace/mission_runner.py" "$MISSION_PATH" \ + --airstack-root "$AIRSTACK_ROOT" + log "mission_runner exited $?" + else + fail "mission file not found: $MISSION (looked under the clone and as an absolute path)" + fi +fi + +# ── 3b. NAS upload + teardown (optional) ─────────────────────────────────── +# With the airlab-storage credential injected and a destination set (--nas-dest, +# else the spec's nas_dest:), rsync results to the NAS. Success → pod exits (GPU +# freed); failure → stay alive so results survive for osmo:fetch. +NAS_YAML_DEST="" +if [ -f "${MISSION_PATH:-}" ]; then + NAS_YAML_DEST="$(python3 -c 'import yaml,sys; d=yaml.safe_load(open(sys.argv[1])) or {}; print(d.get("nas_dest") or "")' "$MISSION_PATH" 2>/dev/null)" +fi +NAS_DEST="${OSMO_MISSION_UPLOAD_DEST:-$NAS_YAML_DEST}" + +if [ "${OSMO_MISSION_NO_UPLOAD:-false}" != "true" ] \ + && [ -n "${AIRLAB_STORAGE_USER:-}" ] && [ -n "${AIRLAB_STORAGE_PASS:-}" ] \ + && [ -n "$NAS_DEST" ]; then + if ! command -v sshpass >/dev/null 2>&1; then + log "installing sshpass (image predates it)" + { apt-get update -qq && apt-get install -y -qq sshpass; } \ + || fail "could not install sshpass — skipping NAS upload" + fi + if command -v sshpass >/dev/null 2>&1; then + nas_host="${AIRLAB_STORAGE_HOST:-airlab-storage.andrew.cmu.edu}" + results_dir="${OSMO_RESULTS_ROOT:-$AIRSTACK_ROOT/osmo/results}" + ssh_cmd="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" + log "NAS upload: ${results_dir}/ → ${nas_host}:${NAS_DEST%/}/" + export SSHPASS="$AIRLAB_STORAGE_PASS" + # NAS share (Synology rsync 3.1.x) rejects --mkpath and can't chmod/chown the + # SMB-backed volume, so skip those attrs; nas_dest base must already exist. + sshpass -e rsync -rltz --partial --timeout=1800 \ + --no-perms --no-owner --no-group --omit-dir-times -e "$ssh_cmd" \ + "${results_dir}/" "${AIRLAB_STORAGE_USER}@${nas_host}:${NAS_DEST%/}/" + nas_rc=$? + unset SSHPASS + if [ "$nas_rc" -eq 0 ]; then + log "NAS upload OK → ${nas_host}:${NAS_DEST} — tearing pod down (GPU freed)" + release_osmo_pipe + exit 0 + fi + fail "NAS upload FAILED (rc=$nas_rc) — keeping pod alive for 'airstack osmo:fetch' / SSH debug" + exec sleep infinity + fi +elif [ "${OSMO_MISSION_NO_UPLOAD:-false}" != "true" ] \ + && [ -n "${AIRLAB_STORAGE_USER:-}" ] && [ -z "$NAS_DEST" ]; then + log "airlab-storage set but no nas_dest / --nas-dest — skipping upload; pod stays alive." +fi + +# ── 4. lifetime ──────────────────────────────────────────────────────────── +if [ "${OSMO_MISSION_KEEP_ALIVE:-true}" = "true" ]; then + log "OSMO_MISSION_KEEP_ALIVE=true — pod stays alive; fetch with 'airstack osmo:fetch'" + exec sleep infinity +fi +log "OSMO_MISSION_KEEP_ALIVE=false — exiting so OSMO uploads /osmo/output and frees the GPU" +release_osmo_pipe +exit 0 diff --git a/osmo/workspace/mission_runner.py b/osmo/workspace/mission_runner.py new file mode 100644 index 000000000..110b8067d --- /dev/null +++ b/osmo/workspace/mission_runner.py @@ -0,0 +1,1478 @@ +#!/usr/bin/env python3 +"""mission_runner.py — batch mission executor for AirStack-on-OSMO pods. + +Reads a declarative mission spec (YAML, see osmo/missions/README.md) and runs +N full iterations of: + + airstack down → airstack up → wait for PX4 ready → start mcap recording + → execute steps (ros2 action goals / topic pubs / service calls / raw + commands / waits) → stop recording → collect bags + container logs → + airstack down + +Artifacts land under one results root per mission run: + + /// + ├── summary.json # per-iteration pass/fail + durations + method + └── iter_001____/ # e.g. iter_001__warehouse__frontier + # (iter_NNN prefix kept; env+method appended) + ├── bags/robot_1/*.mcap # Foxglove-ready (open the .mcap directly) + ├── logs/.log # docker logs snapshot, per container + ├── ready.json # per-robot PX4 readiness timings + └── steps.json # per-step command, output tail, status + +The results root prefers /osmo/output (OSMO uploads that directory to the +workflow's `outputs:` destinations when the task exits) and falls back to +/osmo/results. Either way a symlink is left at +/osmo/results so `airstack osmo:fetch` always finds it. + +Designed to run on the OSMO workspace pod (invoked by entrypoint.sh when +OSMO_MISSION_FILE is set), but has no OSMO dependency: it only needs docker, +python3 + PyYAML, and a checkout with airstack.sh — so it can be tested on +any dev machine that can run `airstack up`. + +Command patterns (ros2 exec into the robot container with a per-robot +ROS_DOMAIN_ID, two-gate PX4 readiness, action-result parsing) mirror +tests/conftest.py and tests/system/test_takeoff_hover_land.py. +""" + +import argparse +import json +import os +import re +import shlex +import signal +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path + +import yaml + +ROS_DISTRO_SETUP = "/opt/ros/jazzy/setup.bash" +DEFAULT_ROBOT_SETUP_BASH = "/root/AirStack/robot/ros_ws/install/setup.bash" +GCS_SETUP_BASH = "/root/AirStack/gcs/ros_ws/install/setup.bash" +ROBOT_CONTAINER_PATTERN = "robot.*desktop" + +# Default topics for `record.scope: gcs` — one recorder in the GCS container +# on domain 0, where each robot's domain_bridge (autonomy_bringup +# onboard_all/config/domain_bridge.yaml) forwards its state topics. All +# robots land in ONE mcap, and replaying it in Foxglove reproduces the live +# GCS view (gcs_visualizer markers are already in the global 'map' frame). +# {robot} expands to robot_1..robot_N and the result is unioned. +DEFAULT_GCS_RECORD_TOPICS = [ + "/tf", + "/tf_static", + "/{robot}/odometry_conversion/odometry", + "/{robot}/interface/mavros/global_position/global", + "/{robot}/trajectory_controller/trajectory_vis", + "/{robot}/global_plan", + "/gcs/robot_markers", + "/gcs/map_origin/location", + "/gcs/map_origin/ground_msl", + "/gcs/{robot}/location", +] + +# Default topics for `record.scope: robot` — one recorder (and one mcap) per +# robot on its own DDS domain. Use this scope for topics that are NOT +# bridged to the GCS (raw sensors, high-rate local topics). +DEFAULT_ROBOT_RECORD_TOPICS = [ + "/tf", + "/tf_static", + "/{robot}/odometry_conversion/odometry", + "/{robot}/interface/mavros/global_position/global", + "/{robot}/global_plan", +] + +# In-container staging dir for bag recordings (docker cp'd out before the +# stack goes down). +BAG_STAGING_DIR = "/tmp/osmo_bags" + +# Tasks the GCS action_relay bridges (gcs/ros_ws/src/action_relay). Goals for +# these can be routed `via: gcs` — published as String JSON on +# //tasks//goal (GCS domain 0), exactly the path Foxglove uses. +GCS_RELAY_TASKS = {"takeoff", "land", "navigate", "fixed_trajectory", + "semantic_search", "exploration"} + +MISSION_DEFAULTS = { + "iterations": 1, + # How many times to (re)run a single iteration that doesn't pass before + # giving up and moving on. 1 = no redo (the historical behavior). >1 = if an + # iteration ends failed/errored (a step up to and including the gating step + # failed — e.g. takeoff never armed, or semantic_search never got feedback), + # tear the stack down and run that same iteration again, up to this many + # total attempts. A `stopped` (manual) or `abort_mission` outcome is never + # retried. Failed attempts' artifacts are preserved under + # iter_NNN_failed_attempt_K so the canonical iter_NNN holds the last attempt. + "iteration_attempts": 1, + # With an `environments:` list: round_robin → iteration i uses + # environments[(i-1) % n], so `iterations` is the TOTAL run count. + # grouped → each environment runs `iterations` times consecutively + # (retro×N, then fireacademy×N, …), so the total is iterations * n. + "environment_order": "round_robin", # round_robin | grouped + "on_step_failure": "abort_iteration", # continue | abort_iteration | abort_mission + "ready": {"timeout_s": 600, "poll_interval_s": 5}, + # scope "gcs": one recorder on GCS domain 0 → one mcap with every robot's + # bridged topics (default). scope "robot": one recorder + one mcap per + # robot on its own domain (for unbridged/high-rate topics). scope "both": + # the GCS recorder AND the per-robot recorders together. + "record": {"enabled": True, "scope": "gcs"}, + # How the stack is brought up. Either (or both): + # services: [isaac-sim, robot-desktop, gcs] → ./airstack.sh up + # (compose auto-enables a named service's profile) + # profiles: [desktop, isaac-sim] → exported as COMPOSE_PROFILES + # With neither, plain `airstack up` with COMPOSE_PROFILES=desktop,isaac-sim. + "stack": {"services": [], "profiles": []}, + # Default route for `action` steps: "gcs" sends goals through the GCS + # action_relay (same path as Foxglove / the GCS panels — exercises the + # full GCS→robot chain incl. the relay's airborne preconditions); + # "robot" sends ros2 action goals directly on the robot's DDS domain. + # Override per step with `via:`. + "command_route": "gcs", + "up_timeout_s": 3600, # first `up` on a fresh pod pulls the full image set + "down_timeout_s": 300, + "robot_setup_bash": DEFAULT_ROBOT_SETUP_BASH, +} + + +class MissionStop(Exception): + """Raised in the main thread on SIGINT/SIGTERM (e.g. `airstack + osmo:stop`): aborts the current step, finalizes recordings, collects + artifacts, and brings the stack down.""" + + +STOP_EVENT = threading.Event() + + +def _stop_handler(signum, frame): + if STOP_EVENT.is_set(): + return # already stopping — don't interrupt artifact collection + STOP_EVENT.set() + raise MissionStop(f"stop requested (signal {signum})") + + +def log(msg): + print(f"[mission {datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True) + + +def utc_stamp(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S") + + +# ── subprocess / docker plumbing ─────────────────────────────────────────── + +def sh(cmd_list, timeout, env=None, cwd=None): + """Run a command, capturing output. Never raises on non-zero exit; a + TimeoutExpired is converted into a synthetic failed result so step + failures stay data, not exceptions.""" + try: + return subprocess.run(cmd_list, capture_output=True, text=True, + timeout=timeout, env=env, cwd=cwd) + except subprocess.TimeoutExpired as e: + return subprocess.CompletedProcess( + cmd_list, returncode=124, + stdout=(e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or ""), + stderr=f"timed out after {timeout}s", + ) + + +def docker_exec(container, cmd, timeout): + return sh(["docker", "exec", container, "bash", "-c", cmd], timeout=timeout) + + +def ros2_env_prefix(setup_bash, domain_id): + # Workspace setup is conditional: the GCS container has a different (or + # no) workspace at the robot path, and plain std_msgs pub/echo only needs + # the distro setup anyway. + return (f"source {ROS_DISTRO_SETUP} && " + f"{{ [ -f {setup_bash} ] && source {setup_bash}; }}; " + f"export ROS_DOMAIN_ID={domain_id}") + + +def ros2_exec(container, ros2_cmd, domain_id, setup_bash, timeout): + return docker_exec(container, f"{ros2_env_prefix(setup_bash, domain_id)} && {ros2_cmd}", + timeout=timeout) + + +def list_containers(name_pattern=None, all_states=False): + cmd = ["docker", "ps", "--format", "{{.Names}}"] + if all_states: + cmd.insert(2, "-a") + if name_pattern: + cmd += ["--filter", f"name={name_pattern}"] + result = sh(cmd, timeout=15) + return [n for n in result.stdout.strip().splitlines() if n] + + +def robot_containers(): + """Running robot containers sorted by replica index (replica n ↔ robot_n).""" + def index(name): + tail = name.rsplit("-", 1)[-1] + return int(tail) if tail.isdigit() else 0 + return sorted(list_containers(ROBOT_CONTAINER_PATTERN), key=index) + + +def gcs_container(): + names = list_containers("gcs") + return names[0] if names else None + + +def stack_containers(): + """Running containers belonging to the AirStack compose stack (robot, gcs, + and any sim variant — isaac-sim / isaac-sim-livestream / ms-airsim).""" + pats = ("airstack", "isaac-sim", "ms-airsim") + return [n for n in list_containers() if any(p in n for p in pats)] + + +# ── mission spec ─────────────────────────────────────────────────────────── + +def load_mission(path): + with open(path, encoding="utf-8") as f: + spec = yaml.safe_load(f) + if not isinstance(spec, dict): + raise ValueError(f"{path}: mission spec must be a YAML mapping") + if not spec.get("steps"): + raise ValueError(f"{path}: mission spec has no 'steps'") + merged = dict(MISSION_DEFAULTS) + for key, default in MISSION_DEFAULTS.items(): + if isinstance(default, dict): + merged[key] = {**default, **(spec.get(key) or {})} + merged.update({k: v for k, v in spec.items() if not isinstance(MISSION_DEFAULTS.get(k), dict)}) + merged.setdefault("name", Path(path).stem) + merged.setdefault("env", {}) + if merged["on_step_failure"] not in ("continue", "abort_iteration", "abort_mission"): + raise ValueError(f"on_step_failure must be continue|abort_iteration|abort_mission, " + f"got {merged['on_step_failure']!r}") + if merged["environment_order"] not in ("round_robin", "grouped"): + raise ValueError(f"environment_order must be round_robin|grouped, " + f"got {merged['environment_order']!r}") + try: + merged["iteration_attempts"] = int(merged["iteration_attempts"]) + except (TypeError, ValueError): + raise ValueError(f"iteration_attempts must be a positive integer, " + f"got {merged['iteration_attempts']!r}") + if merged["iteration_attempts"] < 1: + raise ValueError(f"iteration_attempts must be >= 1, " + f"got {merged['iteration_attempts']}") + if merged["command_route"] not in ("gcs", "robot"): + raise ValueError(f"command_route must be gcs|robot, got {merged['command_route']!r}") + if merged["record"].get("scope", "gcs") not in ("gcs", "robot", "both"): + raise ValueError(f"record.scope must be gcs|robot|both, " + f"got {merged['record'].get('scope')!r}") + envs = merged.get("environments") or [] + if not isinstance(envs, list): + raise ValueError("environments must be a list of environment entries") + for e in envs: + if not isinstance(e, dict) or "name" not in e: + raise ValueError(f"each environments entry must be a mapping with a " + f"'name' key, got {e!r}") + merged["environments"] = envs + for step in merged["steps"]: + action = step.get("action") if isinstance(step, dict) else None + if action: + via = action.get("via", merged["command_route"]) + if via not in ("gcs", "robot"): + raise ValueError(f"action via must be gcs|robot, got {via!r}") + if via == "gcs" and action.get("task") not in GCS_RELAY_TASKS: + raise ValueError( + f"task '{action.get('task')}' is not bridged by the GCS action_relay " + f"({', '.join(sorted(GCS_RELAY_TASKS))}); add `via: robot` to send it " + f"directly on the robot's domain") + if action.get("pass_on_feedback") and via != "gcs": + raise ValueError( + "pass_on_feedback requires via: gcs (feedback liveness is only " + "tracked on the GCS action_relay path)") + return merged + + +def task_action_type(task): + """tasks/semantic_search → task_msgs/action/SemanticSearchTask""" + camel = "".join(part.capitalize() for part in task.split("_")) + return f"task_msgs/action/{camel}Task" + + +def expand(text, n): + """Substitute per-robot placeholders: {n} → robot index, {robot} → robot_.""" + return text.replace("{robot}", f"robot_{n}").replace("{n}", str(n)) + + +def has_placeholder(*strings): + """True if any string contains a per-robot placeholder ({robot} or {n}).""" + return any(s and ("{robot}" in s or "{n}" in s) for s in strings) + + +def step_robots(step_spec, num_robots): + robots = step_spec.get("robots", "all") + if robots == "all": + return list(range(1, num_robots + 1)) + return [int(r) for r in robots] + + +def uses_gcs_route(mission): + """True if any action step routes through the GCS action_relay.""" + for step in mission["steps"]: + action = step.get("action") if isinstance(step, dict) else None + if action and action.get("via", mission["command_route"]) == "gcs": + return True + return False + + +# ── per-iteration environment cycling ────────────────────────────────────── +# An optional top-level `environments:` list lets one mission run several +# scenes. environment_order=round_robin: iteration i uses environments[(i-1) % +# len] (`iterations` = total). environment_order=grouped: each env runs +# `iterations` times in a row (total = iterations * len). Within an entry, +# UPPERCASE keys (ENV_URL, SPAWN_POLY, …) are exported as env vars before +# `airstack up` (so they reach the isaac-sim container); any key can be +# referenced from a step via a {{env.KEY}} placeholder (e.g. the search_area +# polygon, which must track the scene). + +_ENV_PLACEHOLDER = re.compile(r"\{\{\s*env\.([A-Za-z0-9_]+)\s*\}\}") + + +def env_exports(env_entry): + """UPPERCASE keys of an environment entry → env vars for `airstack up`. + Structured values are JSON-encoded so SPAWN_POLY can be written either as a + YAML list or as a JSON string in the mission spec.""" + return {k: (v if isinstance(v, str) else json.dumps(v)) + for k, v in env_entry.items() if k.isupper()} + + +def substitute_env(obj, env_entry): + """Replace {{env.KEY}} placeholders in a (copy of a) step spec with values + from the current environment entry. A string that is *exactly* one + placeholder yields the raw value (so {{env.search_area}} can resolve to a + dict); a placeholder embedded in a larger string is substituted textually.""" + if isinstance(obj, str): + whole = _ENV_PLACEHOLDER.fullmatch(obj.strip()) + if whole: + return _lookup_env(env_entry, whole.group(1)) + + def repl(m): + val = _lookup_env(env_entry, m.group(1)) + return val if isinstance(val, str) else json.dumps(val) + return _ENV_PLACEHOLDER.sub(repl, obj) + if isinstance(obj, dict): + return {k: substitute_env(v, env_entry) for k, v in obj.items()} + if isinstance(obj, list): + return [substitute_env(v, env_entry) for v in obj] + return obj + + +def _lookup_env(env_entry, key): + if key not in env_entry: + raise ValueError(f"{{{{env.{key}}}}} referenced in a step but not found in " + f"environment '{env_entry.get('name')}' " + f"(available keys: {sorted(env_entry)})") + return env_entry[key] + + +# ── iteration naming (env + method in the dir name) ───────────────────────── +# `method:` is an optional free-form label (on an environment entry or at the +# mission top level) for missions that compare algorithms/configurations across +# iterations — it lands in the iteration dir name and summary.json so runs stay +# distinguishable after fetch. + +def _slug(s): + """Filesystem-safe lowercase token.""" + s = re.sub(r"[^A-Za-z0-9._-]+", "-", str(s).strip().lower()) + return s.strip("-") + + +def resolve_method(mission, env_entry): + """Normalized method label. Explicit `method:` on the environment entry + wins, then a mission-level `method:`; 'na' (omitted from names) otherwise.""" + if env_entry and env_entry.get("method"): + return _slug(env_entry["method"]) + if mission.get("method"): + return _slug(mission["method"]) + return "na" + + +def iter_dir_name(i, env_entry, method): + """iter_NNN[__][__]. The iter_NNN prefix is kept so `iter_*` + globs and numeric ordering still work; a trailing method suffix already in + the env name is dropped to avoid e.g. scene_a_frontier__frontier.""" + parts = [f"iter_{i:03d}"] + env_slug = _slug(env_entry["name"]) if env_entry and env_entry.get("name") else "" + if env_slug: + for sep in ("_", "-"): + if method != "na" and env_slug.endswith(f"{sep}{method}"): + env_slug = env_slug[: -len(method) - 1] + break + if env_slug: + parts.append(env_slug) + if method and method != "na": + parts.append(method) + return "__".join(parts) + + +# ── stack lifecycle ──────────────────────────────────────────────────────── + +class Stack: + def __init__(self, airstack_root, mission): + self.root = airstack_root + self.mission = mission + self.env = os.environ.copy() + self.env.update({k: str(v) for k, v in mission["env"].items()}) + self.env.setdefault("AUTOLAUNCH", "true") + self.env.setdefault("NUM_ROBOTS", "1") + # Bring-up selection (see MISSION_DEFAULTS["stack"]): an explicit + # service list is passed to `airstack up `; explicit + # profiles become COMPOSE_PROFILES. With neither, default profiles. + self.services = [str(s) for s in mission["stack"].get("services") or []] + profiles = [str(p) for p in mission["stack"].get("profiles") or []] + if profiles: + self.env["COMPOSE_PROFILES"] = ",".join(profiles) + elif not self.services: + self.env.setdefault("COMPOSE_PROFILES", "desktop,isaac-sim") + self.num_robots = int(self.env["NUM_ROBOTS"]) + self.setup_bash = mission["robot_setup_bash"] + + def apply_env(self, overrides): + """Merge per-iteration env-var overrides (from an `environments:` entry) + into the bring-up environment so the next `airstack up` sees them.""" + for k, v in overrides.items(): + self.env[k] = v if isinstance(v, str) else str(v) + + def _airstack(self, verb, timeout, extra_args=()): + log(f"airstack {verb} {' '.join(extra_args)} " + f"(NUM_ROBOTS={self.env['NUM_ROBOTS']}, " + f"COMPOSE_PROFILES={self.env.get('COMPOSE_PROFILES', '')})") + return sh([str(Path(self.root) / "airstack.sh"), verb, *extra_args], + timeout=timeout, env=self.env, cwd=self.root) + + def up(self): + result = self._airstack("up", self.mission["up_timeout_s"], + extra_args=self.services) + if result.returncode != 0: + raise RuntimeError(f"airstack up failed (exit {result.returncode}):\n" + + "\n".join(result.stdout.splitlines()[-20:]) + + "\n" + "\n".join(result.stderr.splitlines()[-20:])) + deadline = time.time() + 120 + while time.time() < deadline: + containers = robot_containers() + if len(containers) >= self.num_robots: + return containers + time.sleep(3) + raise RuntimeError(f"expected {self.num_robots} robot containers, " + f"found {robot_containers()} after airstack up") + + def down(self): + result = self._airstack("down", self.mission["down_timeout_s"]) + if result.returncode != 0: + log(f"WARN: airstack down exited {result.returncode}") + + def ensure_down(self): + """`airstack status`; if any stack containers exist, `airstack down` + and BLOCK until they're actually gone before returning. + + This is the pre-`up` guard: a fresh `airstack up` must not race a + previous bring-up's containers/network (the baked entrypoint leaves a + stack up on a stale image; the prior iteration leaves one too). + Starting `up` while those still exist causes duplicate-network and + container-name conflicts.""" + self._airstack("status", 60) + existing = stack_containers() + if not existing: + return + log(f"existing stack containers found {existing} — bringing them down first") + self.down() + deadline = time.time() + self.mission["down_timeout_s"] + while time.time() < deadline: + remaining = stack_containers() + if not remaining: + log("stack fully down; safe to bring up") + return + log(f"waiting for teardown: {remaining}") + time.sleep(2) + log(f"WARN: containers still present after down: {stack_containers()} " + f"— proceeding, `up` may conflict") + + def wait_ready(self, container): + """Two sequential gates per robot (same as the system tests): + 1. mavros/state reports connected=True (MAVROS ↔ PX4 heartbeat); + 2. local_position/odom publishes (PX4 EKF converged — the actual + precondition for arming; `connected` alone fires ~25s too early). + Returns {robot_n: seconds_to_ready}; raises on timeout.""" + cfg = self.mission["ready"] + started = time.time() + deadline = started + cfg["timeout_s"] + connected, ready_at = set(), {} + pending = list(range(1, self.num_robots + 1)) + + while pending and time.time() < deadline: + for n in list(pending): + if n not in connected: + r = ros2_exec(container, + f"timeout 5 ros2 topic echo --once --csv " + f"--field connected /robot_{n}/interface/mavros/state", + domain_id=n, setup_bash=self.setup_bash, timeout=15) + if any(line.strip() == "True" for line in r.stdout.splitlines()): + connected.add(n) + else: + continue + r = ros2_exec(container, + f"timeout 5 ros2 topic echo --once " + f"/robot_{n}/interface/mavros/local_position/odom", + domain_id=n, setup_bash=self.setup_bash, timeout=15) + if r.returncode == 0 and "pose:" in r.stdout: + ready_at[n] = round(time.time() - started, 2) + pending.remove(n) + if pending: + log(f"waiting for PX4: connected={sorted(connected)} pending={pending} " + f"elapsed={time.time() - started:.0f}s") + time.sleep(cfg["poll_interval_s"]) + + if pending: + raise RuntimeError(f"robots {pending} not ready within {cfg['timeout_s']}s " + f"(connected so far: {sorted(connected)})") + log(f"PX4 ready: {ready_at}") + + # Gate 3 (only when actions route via the GCS): the per-robot + # action_relay nodes must be up on the GCS before goals are sent — + # the relay's goal subscription is volatile, so a goal published + # before the relay exists is silently lost. + if uses_gcs_route(self.mission): + while time.time() < deadline: + gcs = gcs_container() + if gcs: + r = ros2_exec(gcs, "timeout 10 ros2 node list", + domain_id=0, setup_bash=GCS_SETUP_BASH, timeout=20) + missing = [n for n in range(1, self.num_robots + 1) + if f"action_relay_robot_{n}" not in r.stdout] + if not missing: + ready_at["gcs_relay"] = round(time.time() - started, 2) + log(f"GCS action_relay ready ({ready_at['gcs_relay']}s)") + break + log(f"waiting for GCS action_relay: missing robots {missing}") + else: + log("waiting for gcs container") + time.sleep(cfg["poll_interval_s"]) + if "gcs_relay" not in ready_at: + raise RuntimeError( + f"GCS action_relay not ready within {cfg['timeout_s']}s — " + f"is the gcs service in stack.services/profiles and " + f"AUTOLAUNCH=true? (or set command_route: robot)") + return ready_at + + +# ── bag recording ────────────────────────────────────────────────────────── + +class Recorder: + """`ros2 bag record -s mcap` per the mission's record.scope. + + scope "gcs" (default): a single recorder in the GCS container on domain + 0, where every robot's domain_bridge forwards its state topics — one + mcap holds all robots, and Foxglove replay matches the live GCS view. + + scope "robot": one recorder per robot inside robot container 1, each on + that robot's DDS domain — for topics that aren't bridged to the GCS. + + scope "both": the GCS recorder and the per-robot recorders together — + one mcap with the fleet view from domain 0 plus one raw-data mcap per + robot domain. + + Each recorder is started detached with its PID dropped to a file, and + stopped with SIGTERM so rosbag2 finalizes the mcap cleanly. (Not SIGINT: + jobs backgrounded from a non-interactive shell have SIGINT set to + SIG_IGN, so it would never be delivered; rosbag2 handles SIGTERM the + same way.) Bags are docker cp'd to the host before the stack goes down.""" + + def __init__(self, robot_container, mission, num_robots, setup_bash): + self.robot_container = robot_container + self.cfg = mission["record"] + self.scope = self.cfg.get("scope", "gcs") + self.num_robots = num_robots + self.setup_bash = setup_bash + # (container, domain_id, tag) per active recorder. + self.active = [] + + def _topic_selection(self, robots, scope): + """Build the `ros2 bag record` topic args; `robots` is the list of + robot indices whose {robot}/{n} placeholders to expand (unioned, + order-preserving, deduplicated). `scope` picks the default topic set + for the recorder being started ("gcs" or "robot").""" + if self.cfg.get("all"): + sel = "-a" + # record.exclude: regex of topics to drop from `-a`. + exclude = self.cfg.get("exclude") + if exclude: + sel += f" --exclude-regex {shlex.quote(exclude)}" + return sel + default = (DEFAULT_GCS_RECORD_TOPICS if scope == "gcs" + else DEFAULT_ROBOT_RECORD_TOPICS) + topics = [] + for t in self.cfg.get("topics", default): + if "{robot}" in t or "{n}" in t: + topics.extend(expand(t, n) for n in robots) + else: + topics.append(t) + seen = set() + unique = [t for t in topics if not (t in seen or seen.add(t))] + return " ".join(shlex.quote(t) for t in unique) + + def _start_one(self, container, domain_id, tag, selection, setup_bash): + out_dir = f"{BAG_STAGING_DIR}/{tag}" + log_file = f"{BAG_STAGING_DIR}/record_{tag}.log" + pid_file = f"{BAG_STAGING_DIR}/record_{tag}.pid" + # Create the staging dir in the FOREGROUND, THEN background only the + # recorder. Folding `mkdir` into the same `... &` chain backgrounds the + # mkdir too, so the foreground pidfile write races ahead of the dir + # existing ("record_.pid: No such file or directory"). nohup + + # pidfile lets the recorder outlive this docker exec; topics that + # don't exist yet are fine (rosbag2 waits for them). + inner = ( + f"mkdir -p {BAG_STAGING_DIR} && rm -rf {out_dir}\n" + f"{ros2_env_prefix(setup_bash, domain_id)}\n" + f"nohup ros2 bag record -s mcap -o {out_dir} {selection} " + f"> {log_file} 2>&1 &\n" + f"echo $! > {pid_file}\n" + # Confirm the recorder is actually alive — a bad topic name or a + # missing mcap plugin would make it exit immediately, and a silent + # recording failure is the worst outcome for a mission. + f"sleep 2\n" + f"if kill -0 \"$(cat {pid_file})\" 2>/dev/null; then echo RECORDER_ALIVE; " + f"else echo RECORDER_DEAD; tail -n 5 {log_file} 2>/dev/null; fi" + ) + r = docker_exec(container, inner, timeout=30) + if "RECORDER_ALIVE" in r.stdout: + self.active.append((container, tag)) + n_topics = ("all topics" if selection.startswith("-a") + else f"{len(selection.split())} topics") + log(f"recording [{tag}] in {container} (domain {domain_id}) " + f"→ {out_dir} ({n_topics})") + return True + log(f"WARN: recorder [{tag}] failed to start / exited immediately: " + f"{tail(r.stdout + r.stderr, 6)}") + return False + + def start(self): + if not self.cfg.get("enabled", True): + log("recording disabled by mission spec") + return + robots = list(range(1, self.num_robots + 1)) + failed = [] + if self.scope in ("gcs", "both"): + gcs = gcs_container() + if gcs: + if not self._start_one(gcs, 0, "gcs", + self._topic_selection(robots, "gcs"), + GCS_SETUP_BASH): + failed.append("gcs") + else: + log("WARN: record.scope includes 'gcs' but no gcs container is " + "running — GCS recording skipped (bring up the gcs service " + "or use scope: robot)") + failed.append("gcs (no container)") + if self.scope == "gcs" and not self.cfg.get("required"): + return + if self.scope in ("robot", "both"): + for n in robots: + if not self._start_one(self.robot_container, n, f"robot_{n}", + self._topic_selection([n], "robot"), + self.setup_bash): + failed.append(f"robot_{n}") + # record.required: a mission whose deliverable is the bags shouldn't + # silently fly unrecorded. + if failed and self.cfg.get("required"): + raise RuntimeError( + f"record.required is set and recorder(s) failed to start: " + f"{failed} — aborting iteration") + + def stop(self): + for container, tag in self.active: + docker_exec(container, f""" + pid=$(cat {BAG_STAGING_DIR}/record_{tag}.pid 2>/dev/null) || exit 0 + kill -TERM "$pid" 2>/dev/null || exit 0 + for i in $(seq 1 20); do + kill -0 "$pid" 2>/dev/null || exit 0 + sleep 1 + done + kill -9 "$pid" 2>/dev/null || true + """, timeout=40) + if self.active: + log(f"recorders stopped: {[tag for _, tag in self.active]}") + + def collect(self, dest_dir): + if not self.active: + return + dest_dir.mkdir(parents=True, exist_ok=True) + for container in {c for c, _ in self.active}: + r = sh(["docker", "cp", f"{container}:{BAG_STAGING_DIR}/.", str(dest_dir)], + timeout=1800) + if r.returncode != 0: + log(f"WARN: docker cp of bags from {container} failed: " + f"{r.stderr.strip()[:200]}") + self.active = [] + mcaps = list(dest_dir.rglob("*.mcap")) + log(f"collected {len(mcaps)} mcap file(s) → {dest_dir}") + + +# ── step execution ───────────────────────────────────────────────────────── + +def action_ok(stdout): + """ros2 action send_goal --feedback prints the result as YAML; + AirStack task results carry `success: true` on completion.""" + return "success: true" in stdout + + +def tail(text, lines=15): + return "\n".join((text or "").strip().splitlines()[-lines:]) + + +def _run_one(stack, robot_container, target, cmd, timeout, expect_success): + """Run one resolved `run` command. `target` is already robot-expanded: + pod → on the pod itself (cwd = AirStack root) + gcs → in the gcs container on domain 0 + robot_ → in robot container 1 on robot N's DDS domain + → literal container name (domain 0) + """ + if target == "pod": + r = sh(["bash", "-c", cmd], timeout=timeout, cwd=stack.root) + elif target == "gcs": + g = gcs_container() + if g: + r = ros2_exec(g, cmd, domain_id=0, setup_bash=GCS_SETUP_BASH, + timeout=timeout + 15) + else: + r = subprocess.CompletedProcess(cmd, returncode=1, stdout="", + stderr="no gcs container running") + elif target.startswith("robot_") and target[6:].isdigit(): + n = int(target[6:]) + r = ros2_exec(robot_container, cmd, domain_id=n, + setup_bash=stack.setup_bash, timeout=timeout + 15) + else: + r = docker_exec(target, cmd, timeout=timeout + 15) + ok = (r.returncode == 0) if expect_success else True + return {"target": target, "exit": r.returncode, "ok": ok, + "output_tail": tail(r.stdout + r.stderr)} + + +def _run_one_retry(stack, robot_container, target, cmd, timeout, expect_success, + attempts, retry_delay_s): + """_run_one with up to `attempts` tries (retry_delay_s between failures).""" + res = {"target": target, "exit": 1, "ok": False, + "output_tail": "not attempted — mission stop requested", "attempts": 0} + for attempt in range(1, max(1, attempts) + 1): + if STOP_EVENT.is_set(): + break + res = _run_one(stack, robot_container, target, cmd, timeout, expect_success) + res["attempts"] = attempt + if res["ok"]: + break + if attempt < attempts and not STOP_EVENT.is_set(): + log(f"run [{target}] failed (attempt {attempt}/{attempts}); " + f"retrying in {retry_delay_s}s") + time.sleep(retry_delay_s) + return res + + +def run_step(stack, container, step_spec, step_index): + """Execute one step; returns a result dict with ok: bool.""" + record = {"index": step_index, "spec": step_spec, + "started_at": datetime.now(timezone.utc).isoformat()} + t0 = time.time() + + if "wait" in step_spec: + seconds = float(step_spec["wait"]) + log(f"step {step_index}: wait {seconds}s") + time.sleep(seconds) + record.update(type="wait", ok=True) + + elif "action" in step_spec: + spec = step_spec["action"] + task = spec["task"] + via = spec.get("via", stack.mission["command_route"]) + goal = spec.get("goal", {}) + # Normalize the goal to JSON: dicts directly; strings may be ros2-style + # YAML ("{target_altitude_m: 10}") — YAML-parse then re-dump. JSON is + # what the GCS relay requires, and it's a YAML subset so the direct + # send_goal path accepts it too. + goal_obj = yaml.safe_load(goal) if isinstance(goal, str) else goal + goal_json = json.dumps(goal_obj or {}) + timeout = float(spec.get("timeout_s", 120)) + # Per-robot retries for transient rejections (no GPS fix yet, PX4 + # position not converged, goal lost on the relay's volatile sub). + max_attempts = int(spec.get("attempts", 3)) + retry_delay_s = float(spec.get("retry_delay_s", 10)) + # via: gcs only — no relay_feedback AND no result within this window + # after publishing ⇒ the goal is presumed lost; fail fast and retry. + feedback_timeout_s = int(spec.get("feedback_timeout_s", 15)) + # cancel_on_timeout (via:gcs only): on `timeout_s`, publish a CancelGoal + # to the relay's .../cancel topic and wait up to cancel_grace_s for the + # cancelled result, so an on-cancel finalize (e.g. semantic_search's + # metrics) can run and return. + cancel_on_timeout = bool(spec.get("cancel_on_timeout", False)) + cancel_grace_s = int(spec.get("cancel_grace_s", 120)) + # pass_on_feedback (via:gcs only): treat the step as passed as soon as the + # action is confirmed to have actually started running — i.e. any + # relay_feedback was seen (or a result came back), regardless of whether + # the result is success or failure. The ONLY failure left is "goal lost / + # never ran" (no feedback and no result within the retries), which is what + # then fails the step (→ iteration redo when iteration_attempts>1). Use + # this when you only care that the task ran, not its outcome. + pass_on_feedback = bool(spec.get("pass_on_feedback", False)) + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: action {task} {goal_json} via {via} → robots {robots}") + + if via == "gcs": + gcs = gcs_container() + if not gcs: + record.update(type="action", task=task, via=via, ok=False, + error="no gcs container running — `via: gcs` needs the " + "gcs service up (or use `via: robot`)") + record["duration_s"] = round(time.time() - t0, 2) + log(f"step {step_index}: FAILED (no gcs container)") + return record + + # The result echo must outlive `timeout` to catch the post-cancel + # result. + echo_timeout = int(timeout + cancel_grace_s) if cancel_on_timeout \ + else int(timeout) + + def send(n): + # Foxglove's path: String JSON on //tasks//goal, + # result on .../relay_result. relay_result is latched, so a + # fresh subscriber gets the PREVIOUS goal's result — count + # messages before publishing and only accept a NEW one. + # relay_feedback is the liveness signal: nothing there (and + # no result) within feedback_timeout_s ⇒ goal lost, bail out + # so the retry loop re-sends. + base = f"/robot_{n}/tasks/{task}" + result_file = f"/tmp/relay_result_{task}_{n}.out" + fb_file = f"/tmp/relay_feedback_{task}_{n}.out" + msg_yaml = json.dumps({"data": expand(goal_json, n)}) + cancel_yaml = json.dumps({"data": "osmo: timeout cancel"}) + script = ( + f"rm -f {result_file} {fb_file}\n" + f"touch {result_file} {fb_file}\n" + f"( timeout {echo_timeout} ros2 topic echo --field data " + f"{base}/relay_result >> {result_file} 2>&1 ) &\n" + f"sub=$!\n" + f"( timeout {echo_timeout} ros2 topic echo --field data " + f"{base}/relay_feedback >> {fb_file} 2>&1 ) &\n" + f"fb_sub=$!\n" + f"sleep 3\n" + f"pre=$(grep -c '^---' {result_file})\n" + # -w 1: wait for the relay's (VOLATILE) goal sub to match + # before publishing; --keep-alive 3: outlive the publish so + # the RELIABLE handshake flushes (0.1s default drops goals). + f"ros2 topic pub --once -w 1 --keep-alive 3 " + f"{base}/goal std_msgs/msg/String " + f"{shlex.quote(msg_yaml)} > /dev/null\n" + f"sent=$(date +%s)\n" + f"fb_seen=0\n" + f"cancelled=0\n" + f"deadline=$(( sent + {int(timeout)} ))\n" + f"while :; do\n" + f" cur=$(grep -c '^---' {result_file})\n" + f" if [ \"$cur\" -gt \"$pre\" ]; then break; fi\n" + f" now=$(date +%s)\n" + f" if [ \"$fb_seen\" -eq 0 ] && " + f"[ \"$(grep -c '^---' {fb_file})\" -gt 0 ]; then fb_seen=1; fi\n" + f" if [ \"$cancelled\" -eq 0 ] && [ \"$fb_seen\" -eq 0 ] && " + f"[ $(( now - sent )) -ge {feedback_timeout_s} ]; then\n" + f" kill $sub $fb_sub 2>/dev/null\n" + f" echo 'no relay_feedback within {feedback_timeout_s}s " + f"of goal publish — goal presumed lost'\n" + f" exit 0\n" + f" fi\n" + f" if [ $now -ge $deadline ]; then\n" + # Deadline: cancel + keep waiting if cancel_on_timeout, else stop. + f" if [ {1 if cancel_on_timeout else 0} -eq 1 ] && " + f"[ \"$cancelled\" -eq 0 ]; then\n" + f" echo 'osmo: {int(timeout)}s timeout — sending cancel'\n" + f" ros2 topic pub --once -w 1 --keep-alive 3 " + f"{base}/cancel std_msgs/msg/String " + f"{shlex.quote(cancel_yaml)} > /dev/null\n" + f" cancelled=1\n" + f" deadline=$(( now + {cancel_grace_s} ))\n" + f" else\n" + f" break\n" + f" fi\n" + f" fi\n" + f" kill -0 $sub 2>/dev/null || break\n" + f" sleep 1\n" + f"done\n" + f"kill $sub $fb_sub 2>/dev/null\n" + # Surface whether the action ever produced feedback (it ran), + # so the runner can pass on liveness when pass_on_feedback is + # set. The early no-feedback bail above exits before here, so + # this only prints when feedback was genuinely observed. + f"if [ \"$fb_seen\" -eq 1 ]; then echo OSMO_FEEDBACK_SEEN; fi\n" + f"cur=$(grep -c '^---' {result_file})\n" + f"if [ \"$cur\" -gt \"$pre\" ]; then " + f"grep -v '^---' {result_file} | tail -n 1; " + f"else echo 'no relay_result within {echo_timeout}s'; fi" + ) + r = ros2_exec(gcs, script, domain_id=0, setup_bash=GCS_SETUP_BASH, + timeout=int(echo_timeout + 30)) + # For cancel_on_timeout, a returned (success=false) cancel result + # is the intended outcome — treat it as ok so the step doesn't + # retry the search or trip on_step_failure. + got_result = '"success"' in r.stdout + feedback_seen = 'OSMO_FEEDBACK_SEEN' in r.stdout + ok = ('"success": true' in r.stdout + or (cancel_on_timeout and got_result)) + # pass_on_feedback: ran-at-all (feedback seen, or any result + # came back) is enough — outcome doesn't matter. + if pass_on_feedback: + ok = ok or feedback_seen or got_result + return {"exit": r.returncode, "ok": ok, + "feedback_seen": feedback_seen, + "output_tail": tail(r.stdout + r.stderr)} + else: + action_type = spec.get("type", task_action_type(task)) + + def send(n): + cmd = (f"ros2 action send_goal --feedback /robot_{n}/tasks/{task} " + f"{action_type} {shlex.quote(expand(goal_json, n))}") + r = ros2_exec(container, cmd, domain_id=n, setup_bash=stack.setup_bash, + timeout=int(timeout + 15)) + return {"exit": r.returncode, "ok": action_ok(r.stdout), + "output_tail": tail(r.stdout + r.stderr)} + + def send_with_retry(n): + res = {"ok": False, "exit": -1, + "output_tail": "not attempted — mission stop requested"} + attempt = 0 + for attempt in range(1, max_attempts + 1): + if STOP_EVENT.is_set(): + break + log(f"step {step_index}: sending {task} to robot_{n} " + f"(attempt {attempt}/{max_attempts})") + res = send(n) + if res["ok"]: + log(f"step {step_index}: {task} robot_{n} succeeded" + + (f" on attempt {attempt}" if attempt > 1 else "")) + break + log(f"step {step_index}: {task} robot_{n} attempt {attempt} " + f"FAILED: {tail(res.get('output_tail', ''), 1)}") + if attempt < max_attempts and not STOP_EVENT.is_set(): + time.sleep(retry_delay_s) + res["attempts"] = attempt + return n, res + + # No context manager: on MissionStop the `with` form would block in + # shutdown(wait=True) until the in-flight docker execs hit their + # timeouts. The workers die when `airstack down` kills the containers. + pool = ThreadPoolExecutor(max_workers=len(robots)) + try: + results = dict(pool.map(send_with_retry, robots)) + finally: + pool.shutdown(wait=False, cancel_futures=True) + record.update(type="action", task=task, via=via, per_robot=results, + ok=all(v["ok"] for v in results.values())) + + elif "run" in step_spec: + spec = step_spec["run"] + cmd = spec["cmd"] + timeout = float(spec.get("timeout_s", 60)) + expect = spec.get("expect_success", True) + attempts = int(spec.get("attempts", 1)) + retry_delay_s = float(spec.get("retry_delay_s", 10)) + # `container` may reference {n}/{robot} to fan out over robots. If it's + # omitted, default to robot_{n} when the command is per-robot (has a + # placeholder) and robot_1 otherwise. A step fans out over `robots` + # (default all) iff its command or container references {n}/{robot}. + target = spec.get("container") or ("robot_{n}" if has_placeholder(cmd) else "robot_1") + if has_placeholder(cmd, target): + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: run [{target}] {cmd} → robots {robots}") + results = {} + for n in robots: + results[n] = _run_one_retry(stack, container, expand(target, n), + expand(cmd, n), timeout, expect, + attempts, retry_delay_s) + record.update(type="run", per_robot=results, + ok=all(v["ok"] for v in results.values())) + else: + log(f"step {step_index}: run [{target}] {cmd}") + res = _run_one_retry(stack, container, target, cmd, timeout, expect, + attempts, retry_delay_s) + record.update(type="run", **res) + + elif "topic_pub" in step_spec: + spec = step_spec["topic_pub"] + msg = spec.get("msg", {}) + msg_str = msg if isinstance(msg, str) else json.dumps(msg) + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: topic_pub {spec['topic']} → robots {robots}") + results = {} + for n in robots: + cmd = (f"ros2 topic pub --once {expand(spec['topic'], n)} " + f"{spec['type']} {shlex.quote(expand(msg_str, n))}") + r = ros2_exec(container, cmd, domain_id=n, setup_bash=stack.setup_bash, + timeout=float(spec.get("timeout_s", 30)) + 15) + results[n] = {"exit": r.returncode, "ok": r.returncode == 0, + "output_tail": tail(r.stdout + r.stderr)} + record.update(type="topic_pub", per_robot=results, + ok=all(v["ok"] for v in results.values())) + + elif "service_call" in step_spec: + spec = step_spec["service_call"] + req = spec.get("request", {}) + req_str = req if isinstance(req, str) else json.dumps(req) + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: service_call {spec['service']} → robots {robots}") + results = {} + for n in robots: + cmd = (f"ros2 service call {expand(spec['service'], n)} " + f"{spec['type']} {shlex.quote(expand(req_str, n))}") + r = ros2_exec(container, cmd, domain_id=n, setup_bash=stack.setup_bash, + timeout=float(spec.get("timeout_s", 30)) + 15) + results[n] = {"exit": r.returncode, "ok": r.returncode == 0, + "output_tail": tail(r.stdout + r.stderr)} + record.update(type="service_call", per_robot=results, + ok=all(v["ok"] for v in results.values())) + + else: + record.update(type="unknown", ok=False, + error=f"unrecognized step keys: {sorted(step_spec)}") + + record["duration_s"] = round(time.time() - t0, 2) + log(f"step {step_index}: {'OK' if record['ok'] else 'FAILED'} " + f"({record['duration_s']}s)") + return record + + +# ── artifacts ────────────────────────────────────────────────────────────── + +# Navigation / control-stack node-name globs. These log via rcl to ~/.ros/log +# (not /tmp), so they're collected separately — they're what reveals why a +# planner's plan isn't becoming motion (e.g. a robot stuck mid-mission). Task +# nodes are here too so their rcl log is captured as a SECOND path independent +# of the /tmp tee (TEE_LOG_GLOBS) — if one collection path misses, the other +# still shows why the drone wasn't commanded. +CONTROL_STACK_GLOBS = ("droan_gl*", "droan_local_planner*", "trajectory_controller*", + "fixed_trajectory*", "takeoff_landing*", "mavros*", + "semantic_search_task*") + + +# /tmp tee logs written by task-spawned subprocesses and the GCS +# relay/gossip/ddsrouter — the stdout that rcl logging doesn't capture. +TEE_LOG_GLOBS = ("/tmp/gossip_*.log", "/tmp/ddsrouter_*.log", "/tmp/relay_*.log") + + +def _copy_tmp_tee_logs(dest_dir): + """Copy the raw /tmp tee logs from each container into //. + Idempotent (docker cp overwrites), so it is safe to call repeatedly — the + snapshot loop calls it periodically so a hard pod cancel mid-iteration still + leaves the most recent stdout tees on disk.""" + targets = list(robot_containers()) + gcs = gcs_container() + if gcs: + targets.append(gcs) + total = 0 + for name in targets: + r = docker_exec(name, "ls " + " ".join(TEE_LOG_GLOBS) + " 2>/dev/null", + timeout=15) + files = [f for f in r.stdout.split() if f] + if not files: + continue + out = dest_dir / name + out.mkdir(parents=True, exist_ok=True) + for f in files: + sh(["docker", "cp", f"{name}:{f}", str(out)], timeout=120) + total += len(files) + return total + + +def snapshot_task_logs(dest_dir): + """Copy the raw gossip/ddsrouter/relay tees (written to /tmp inside each + container) into //, plus the navigation / + control-stack node logs that rcl writes under ~/.ros/log rather than /tmp. + Includes the GCS container so the GCS-side gossip ddsrouter log is captured.""" + targets = list(robot_containers()) + gcs = gcs_container() + if gcs: + targets.append(gcs) + _copy_tmp_tee_logs(dest_dir) + name_pred = " -o ".join(f"-name '{g}'" for g in CONTROL_STACK_GLOBS) + tar = "/tmp/control_stack_logs.tar.gz" + for name in targets: + r = docker_exec(name, "ls " + " ".join(TEE_LOG_GLOBS) + " 2>/dev/null", + timeout=15) + files = [f for f in r.stdout.split() if f] + out = dest_dir / name + made = docker_exec( + name, + f"f=$(find /root/.ros/log -type f \\( {name_pred} \\) 2>/dev/null); " + f"[ -n \"$f\" ] && tar czf {tar} $f 2>/dev/null && echo COLLECTED", + timeout=30) + if "COLLECTED" in (made.stdout or ""): + out.mkdir(parents=True, exist_ok=True) + sh(["docker", "cp", f"{name}:{tar}", + str(out / "control_stack_logs.tar.gz")], timeout=120) + files = files + ["control_stack_logs.tar.gz"] + if files: + log(f"collected {len(files)} node log(s) from {name}") + + +def snapshot_container_logs(dest_dir): + dest_dir.mkdir(parents=True, exist_ok=True) + pats = ("airstack", "isaac-sim", "ms-airsim") + names = [n for n in list_containers(all_states=True) + if any(p in n for p in pats)] + for name in names: + r = sh(["docker", "logs", name], timeout=120) + (dest_dir / f"{name}.log").write_text( + (r.stdout or "") + (("\n--- stderr ---\n" + r.stderr) if r.stderr else ""), + encoding="utf-8") + + +def write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, default=str) + "\n", encoding="utf-8") + + +def resolve_results_root(airstack_root): + """Prefer /osmo/output (auto-uploaded by OSMO `outputs:` when the task + exits); keep /osmo/results working in both cases so + `airstack osmo:fetch` has one stable path to rsync.""" + fallback = Path(airstack_root) / "osmo" / "results" + override = os.environ.get("OSMO_RESULTS_ROOT") + if override: + root = Path(override) + root.mkdir(parents=True, exist_ok=True) + return root + osmo_output = Path(os.environ.get("OSMO_OUTPUT_DIR", "/osmo/output")) + if osmo_output.is_dir() and os.access(osmo_output, os.W_OK): + root = osmo_output / "airstack-mission-results" + root.mkdir(parents=True, exist_ok=True) + if not fallback.exists(): + fallback.parent.mkdir(parents=True, exist_ok=True) + fallback.symlink_to(root) + return root + fallback.mkdir(parents=True, exist_ok=True) + return fallback + + +# ── main loop ────────────────────────────────────────────────────────────── + +# Per-container health snapshot. Sections are emitted on tagged lines so the +# multi-line ps output stays parseable. cgroup paths are v2; absent on v1 (the +# fields just come back empty). +_SNAP_CMD = r""" +echo "T|$(date '+%F %T')" +echo "DDS|procs=$(pgrep -c ddsrouter 2>/dev/null) estab=$(ss -Htn 2>/dev/null | grep -c ESTAB) ddstcp=$(ss -Htnp 2>/dev/null | grep -c ddsrouter) cpu=$(ps -C ddsrouter -o %cpu= 2>/dev/null | tr '\n' ',')" +echo "MEM|cur=$(cat /sys/fs/cgroup/memory.current 2>/dev/null) max=$(cat /sys/fs/cgroup/memory.max 2>/dev/null) oom_kill=$(awk '/^oom_kill /{print $2}' /sys/fs/cgroup/memory.events 2>/dev/null)" +echo "PS|" +ps -eo rss=,args= --sort=-rss 2>/dev/null | head -12 +""" + +# Processes whose early death we want to catch. Matched as substrings of the +# full command line (python nodes all run as `python3 ...`). Extend per-mission +# with any long-running node whose silent crash would otherwise only show up as +# a stuck drone. +_WATCH_PROCS = ("ddsrouter", "gossip") + + +def _parse_snapshot(out): + ts = dds = mem = "" + oom = None + ps_rows = [] + in_ps = False + for ln in out.splitlines(): + if ln.startswith("T|"): + ts = ln[2:].strip() + elif ln.startswith("DDS|"): + dds = ln[4:].strip() + elif ln.startswith("MEM|"): + mem = ln[4:].strip() + m = re.search(r"oom_kill=(\d+)", mem) + oom = int(m.group(1)) if m else None + elif ln.startswith("PS|"): + in_ps = True + elif in_ps: + p = ln.split(None, 1) + if len(p) == 2 and p[0].isdigit(): + ps_rows.append((int(p[0]), p[1].strip())) + return ts, dds, mem, oom, ps_rows + + +def _format_resources(mem, ps_rows): + def mb(v): + return f"{int(v) // 1048576}MB" if v.isdigit() else (v or "?") + m = re.search(r"cur=(\S+) max=(\S+)", mem) + memstr = f"mem={mb(m.group(1))}/{mb(m.group(2))}" if m else mem + oom = re.search(r"oom_kill=\S+", mem) + key = {k: 0 for k in _WATCH_PROCS} + for rss, args in ps_rows: + for k in _WATCH_PROCS: + if k in args: + key[k] += rss + keystr = " ".join(f"{k}={v // 1024}MB" for k, v in key.items()) + top = [] + for rss, args in ps_rows[:5]: + label = next((k for k in _WATCH_PROCS if k in args), + args.split()[0].rsplit("/", 1)[-1][:16]) + top.append(f"{label}:{rss // 1024}MB") + return f"{memstr} {oom.group(0) if oom else ''} | {keystr} | top: {' '.join(top)}", key + + +def _capture_oom_dmesg(iter_dir, container, ts, why): + """Append the kernel OOM-killer trail (host ring buffer, readable from the + privileged container) when a watched process dies or oom_kill increments.""" + r = docker_exec( + container, + "dmesg -T 2>/dev/null | grep -iE 'killed process|out of memory|oom-kill|" + "invoked oom|memory cgroup' | tail -40", timeout=15) + txt = (r.stdout or "").strip() + block = (f"### {container} @ {ts} — {why}\n" + f"{txt or '(dmesg unavailable in container — run host: dmesg -T | grep -i oom)'}\n") + with open(iter_dir / "oom_dmesg.log", "a", encoding="utf-8") as f: + f.write(block) + + +def transport_snapshot_loop(iter_dir, stop_event, interval_s=5): + """Periodic per-container health snapshot for debugging early process death: + - transport.log : ddsrouter procs + domain-99 TCP (gossip dropout) + - resources.log : cgroup memory, oom_kill counter, watched + heaviest procs + - oom_dmesg.log : kernel OOM trail, captured when a watched process + vanishes or the cgroup oom_kill counter increments + """ + iter_dir.mkdir(parents=True, exist_ok=True) + tlog, rlog = iter_dir / "transport.log", iter_dir / "resources.log" + prev_oom, prev_key = {}, {} + # Periodically copy the /tmp tee logs into iter_dir/logs so a hard pod cancel + # (SIGKILL — run_iteration's finally never runs) still leaves recent + # stdout tees. ~every 30s regardless of the health interval. + tee_every = max(1, round(30 / interval_s)) + tick = 0 + while not stop_event.is_set(): + targets = list(robot_containers()) + gcs = gcs_container() + if gcs: + targets.append(gcs) + tlines, rlines = [], [] + for name in targets: + r = docker_exec(name, _SNAP_CMD, timeout=15) + ts, dds, mem, oom, ps_rows = _parse_snapshot(r.stdout or "") + if not ts: + continue + tlines.append(f"{name} [{ts}] {dds}") + res, key = _format_resources(mem, ps_rows) + rlines.append(f"{name} [{ts}] {res}") + if oom is not None: + if prev_oom.get(name, oom) < oom: + _capture_oom_dmesg(iter_dir, name, ts, f"cgroup oom_kill -> {oom}") + prev_oom[name] = oom + if ps_rows: # only trust presence when we got a real ps sample + for k in _WATCH_PROCS: + if prev_key.get((name, k), 0) > 0 and key[k] == 0: + _capture_oom_dmesg(iter_dir, name, ts, f"{k} vanished") + prev_key[(name, k)] = key[k] + if tlines: + with open(tlog, "a", encoding="utf-8") as f: + f.write("\n".join(tlines) + "\n") + if rlines: + with open(rlog, "a", encoding="utf-8") as f: + f.write("\n".join(rlines) + "\n") + if tick % tee_every == 0: + try: + _copy_tmp_tee_logs(iter_dir / "logs") + except Exception as e: + log(f"WARN: periodic tee-log copy failed: {e}") + tick += 1 + stop_event.wait(interval_s) + + +def run_iteration(stack, mission, iter_dir): + """One full up → ready → record → steps → collect → down cycle. + Returns the iteration summary dict; never raises (failures are data).""" + summary = {"status": "passed", "steps_ok": 0, "steps_failed": 0} + recorder = None + container = None + snap_stop = None + t0 = time.time() + try: + # `airstack status`; if a stack is already up (baked entrypoint on a + # stale image, or the previous iteration), down it and wait for the + # containers to fully disappear before bringing up — otherwise the + # `up` races leftover containers/network. + stack.ensure_down() + containers = stack.up() + container = containers[0] + summary["up_duration_s"] = round(time.time() - t0, 2) + + snap_stop = threading.Event() + threading.Thread(target=transport_snapshot_loop, + args=(iter_dir, snap_stop), daemon=True).start() + + ready_at = stack.wait_ready(container) + write_json(iter_dir / "ready.json", ready_at) + + recorder = Recorder(container, mission, stack.num_robots, stack.setup_bash) + recorder.start() + + steps = [] + for i, step_spec in enumerate(mission["steps"], start=1): + record = run_step(stack, container, step_spec, i) + steps.append(record) + if record["ok"]: + summary["steps_ok"] += 1 + continue + summary["steps_failed"] += 1 + # An `optional: true` step records its failure but never fails the + # iteration (so it doesn't trip on_step_failure or an iteration redo) + # — e.g. a post-search land after the gating step already ran. + if isinstance(step_spec, dict) and step_spec.get("optional"): + log(f"step {i}: failed but marked optional — continuing") + summary.setdefault("optional_failures", []).append(i) + continue + summary["status"] = "failed" + policy = mission["on_step_failure"] + if policy == "continue": + continue + if policy == "abort_mission": + summary["abort_mission"] = True + break + write_json(iter_dir / "steps.json", steps) + + except MissionStop as e: + summary["status"] = "stopped" + summary["abort_mission"] = True + log(f"STOP: {e} — finalizing recordings and collecting artifacts") + + except Exception as e: + summary["status"] = "error" + summary["error"] = str(e) + log(f"ERROR: iteration aborted: {e}") + + finally: + if snap_stop is not None: + snap_stop.set() + # Artifact collection happens even on failure — a failed flight's + # bag is usually the most interesting one. + if recorder is not None: + recorder.stop() + recorder.collect(iter_dir / "bags") + if container is not None or list_containers("airstack", all_states=True): + snapshot_container_logs(iter_dir / "logs") + snapshot_task_logs(iter_dir / "logs") + stack.down() + summary["duration_s"] = round(time.time() - t0, 2) + write_json(iter_dir / "iteration.json", summary) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("mission_file", help="Path to mission spec YAML") + parser.add_argument("--airstack-root", + default=os.environ.get("AIRSTACK_ROOT", "/root/AirStack")) + parser.add_argument("--dry-run", action="store_true", + help="Validate the spec and print the plan without touching docker") + args = parser.parse_args() + + mission = load_mission(args.mission_file) + stack = Stack(args.airstack_root, mission) + + # Graceful stop: SIGINT/SIGTERM (Ctrl-C locally, `airstack osmo:stop` on + # a pod) aborts the current step but still finalizes the mcaps, collects + # bags + logs, and brings the stack down. + signal.signal(signal.SIGINT, _stop_handler) + signal.signal(signal.SIGTERM, _stop_handler) + + log(f"mission '{mission['name']}': {mission['iterations']} iteration(s), " + f"{len(mission['steps'])} step(s), {stack.num_robots} robot(s)" + + (f", up to {mission['iteration_attempts']} attempt(s)/iteration" + if mission["iteration_attempts"] > 1 else "")) + if args.dry_run: + print(yaml.safe_dump(mission, sort_keys=False)) + return 0 + + results_root = resolve_results_root(args.airstack_root) + run_dir = results_root / mission["name"] / utc_stamp() + run_dir.mkdir(parents=True) + log(f"results → {run_dir}") + + environments = mission.get("environments") or [] + n_iter = mission["iterations"] + # Build the per-iteration environment schedule. round_robin: cycle envs, + # `iterations` is the total. grouped: run each env `iterations` times in a + # row, so the total is iterations * len(environments). + if not environments: + schedule = [None] * n_iter + elif mission["environment_order"] == "grouped": + schedule = [env for env in environments for _ in range(n_iter)] + else: + schedule = [environments[i % len(environments)] for i in range(n_iter)] + total = len(schedule) + if environments: + log(f"{mission['environment_order']} over {len(environments)} " + f"environment(s) → {total} iteration(s): " + f"{[e['name'] for e in schedule]}") + + iterations = [] + max_attempts = mission["iteration_attempts"] + for i, env_entry in enumerate(schedule, start=1): + log(f"━━━ iteration {i}/{total} ━━━") + method = resolve_method(mission, env_entry) + iter_dir = run_dir / iter_dir_name(i, env_entry, method) + # Apply this iteration's environment (scene URL + spawn area as env vars + # for `airstack up`; search_area et al. templated into steps). + if env_entry is not None: + log(f"environment: {env_entry['name']} " + f"(ENV_URL={env_entry.get('ENV_URL')})") + stack.apply_env(env_exports(env_entry)) + iter_mission = {**mission, "steps": substitute_env(mission["steps"], env_entry)} + else: + iter_mission = mission + + # Iteration redo: re-run the SAME iteration up to iteration_attempts + # times until it passes. A passed run, a manual stop, or an explicit + # abort_mission is never retried; a failed/errored run is (the stack is + # already down by run_iteration's finally, so each attempt is a clean + # down→up→fly→down cycle — and with a fixed SPAWN_SEED the spawn layout + # is reproduced exactly). + attempt = 0 + while True: + attempt += 1 + if attempt > 1: + log(f"iteration {i}: attempt {attempt}/{max_attempts}") + summary = run_iteration(stack, iter_mission, iter_dir) + done = (summary["status"] in ("passed", "stopped") + or summary.get("abort_mission") + or attempt >= max_attempts) + if done: + break + # Failed with retries left: stash this attempt's artifacts so the + # canonical iter_NNN ends up holding the final attempt. + failed_dir = run_dir / f"{iter_dir.name}_failed_attempt_{attempt}" + try: + if iter_dir.exists(): + iter_dir.rename(failed_dir) + log(f"iteration {i} {summary['status']} on attempt {attempt}" + f"/{max_attempts}; artifacts → {failed_dir.name}; redoing") + except OSError as e: + log(f"WARN: could not stash failed attempt dir ({e}); " + f"redoing into {iter_dir.name}") + + summary["iteration"] = i + summary["attempts"] = attempt + summary["method"] = method + if env_entry is not None: + summary["environment"] = env_entry["name"] + iterations.append(summary) + write_json(run_dir / "summary.json", + {"mission": mission["name"], "mission_file": args.mission_file, + "iterations": iterations}) + if summary.get("abort_mission"): + log("stopping remaining iterations" + + (" (manual stop)" if summary["status"] == "stopped" else + " (on_step_failure=abort_mission)")) + break + + passed = sum(1 for s in iterations if s["status"] == "passed") + log(f"mission complete: {passed}/{len(iterations)} iteration(s) passed; " + f"results in {run_dir}") + return 0 if passed == len(iterations) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/robot/docker/Dockerfile.robot b/robot/docker/Dockerfile.robot index 92fd116fe..b97029f06 100644 --- a/robot/docker/Dockerfile.robot +++ b/robot/docker/Dockerfile.robot @@ -167,6 +167,12 @@ RUN pip3 install --break-system-packages --ignore-installed \ kornia \ typeguard==2.13.3 +# Keep pytest < 8.1. ROS Jazzy launch_testing still implements +# pytest_pycollect_makemodule(path=...), which pluggy rejects after pytest 8.1 +# removed the py.path hook argument (PluginValidationError on colcon test). +RUN python3 -m pip install --no-cache-dir --break-system-packages \ + "pytest>=7.4,<8.1" + # Install MACVO Python dependencies (skipped if SKIP_MACVO=true) RUN if [ "${SKIP_MACVO}" != "true" ]; then \ pip3 install --break-system-packages \ diff --git a/robot/docker/docker-compose.yaml b/robot/docker/docker-compose.yaml index 31b3e368a..cc2bbb6c2 100644 --- a/robot/docker/docker-compose.yaml +++ b/robot/docker/docker-compose.yaml @@ -17,8 +17,13 @@ services: ROS_DISTRO: jazzy tags: - *desktop_image + # Floating tag republished by docker-build.yml on main/develop. The + # versioned tag above never exists yet on a PR (check-version-increment + # forces VERSION up), so it can only ever be a cache miss. + - &desktop_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-x86-64_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *desktop_image + - *desktop_cache # we use tmux sd-keys so that the session stays alive environment: - ROBOT_NAME_SOURCE=container_name # see .bashrc @@ -126,8 +131,10 @@ services: ROS_DISTRO: jazzy tags: - *voxl_image + - &voxl_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-voxl_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *voxl_image + - *voxl_cache environment: - ROBOT_NAME_SOURCE=hostname # see .bashrc - AUTOLAUNCH=${AUTOLAUNCH:-true} @@ -158,8 +165,10 @@ services: DUSTYNV_IMAGE: dustynv/ros:jazzy-ros-base-r36.4.0-cu128-24.04 tags: - *l4t_stack_base_image + - &l4t_stack_base_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-l4t-stack-base_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *l4t_stack_base_image + - *l4t_stack_base_cache # =================================================================================================================== # for running on an NVIDIA jetson (linux for tegra) device @@ -184,9 +193,12 @@ services: ROS_DISTRO: jazzy tags: - *l4t_image + - &l4t_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-l4t_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *l4t_image + - *l4t_cache - *l4t_stack_base_image + - *l4t_stack_base_cache # we use tmux send-keys so that the session stays alive ipc: host command: > @@ -246,8 +258,12 @@ services: L4T_MINOR: 4 L4T_PATCH: 0 IMAGE_NAME: dustynv/ros:jazzy-desktop-r36.4.0-cu128-24.04 + tags: + - *zed_l4t_image + - &zed_l4t_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_zed-l4t-36-4-0_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *zed_l4t_image + - *zed_l4t_cache command: > bash -c "ssh service restart; tmux new -d -s zed_driver && diff --git a/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg index 55f87f13c..7e02f4b35 100644 --- a/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg +++ b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg @@ -10,3 +10,13 @@ python_classes = Test* python_functions = test_* markers = unit: Hermetic unit tests (no ROS stack required) + linter: ament copyright/flake8/pep257 (run separately, not via colcon test) + copyright: ament_copyright + flake8: ament_flake8 + pep257: ament_pep257 +# colcon test / PYTEST_ADDOPTS -m is dropped by ament pytest. Ignore linter +# modules here so only unit tests run. +addopts = + --ignore=test/test_copyright.py + --ignore=test/test_flake8.py + --ignore=test/test_pep257.py diff --git a/robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/conftest.py b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/conftest.py new file mode 100644 index 000000000..c985d5923 --- /dev/null +++ b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/conftest.py @@ -0,0 +1,7 @@ +# Skip ament linter modules during pytest/colcon test. +# PYTEST_ADDOPTS -m is not forwarded by ament pytest; collect_ignore is. +collect_ignore = [ + "test_copyright.py", + "test_flake8.py", + "test_pep257.py", +] diff --git a/simulation/isaac-sim/docker/Dockerfile.isaac-ros b/simulation/isaac-sim/docker/Dockerfile.isaac-ros index 0dca11fb7..69bbd8117 100644 --- a/simulation/isaac-sim/docker/Dockerfile.isaac-ros +++ b/simulation/isaac-sim/docker/Dockerfile.isaac-ros @@ -154,9 +154,23 @@ RUN sed -i \ 's|param set-default IMU_INTEG_RATE 250|param set-default IMU_INTEG_RATE ${PX4_IMU_INTEG_RATE:-250}|' \ /isaac-sim/PX4-Autopilot/ROMFS/px4fmu_common/init.d-posix/px4-rc.simulator -# install px4 dependencies and build -RUN cd PX4-Autopilot && \ - DEBIAN_FRONTEND=noninteractive ./Tools/setup/ubuntu.sh +# Install PX4 host deps and build SITL. +# Match ms-airsim: skip NuttX + Gazebo — Isaac Sim is the simulator, and those +# toolchains are heavy. PX4's ubuntu.sh still apt-installs +# software-properties-common whenever /.dockerenv is present; on the nvcr.io +# Isaac base that package's configure step races with a half-configured +# ca-certificates/launchpadlib chain and fails with dpkg exit 100. Reconfigure +# ca-certificates first and drop software-properties-common from the script +# (add-apt-repository is unused on the --no-nuttx/--no-sim-tools path). +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates && \ + update-ca-certificates && \ + dpkg --configure -a || true && \ + sed -i '/software-properties-common/d' PX4-Autopilot/Tools/setup/ubuntu.sh && \ + cd PX4-Autopilot && \ + DEBIAN_FRONTEND=noninteractive ./Tools/setup/ubuntu.sh --no-nuttx --no-sim-tools && \ + rm -rf /var/lib/apt/lists/* + # build px4 RUN cd PX4-Autopilot && \ make px4_sitl diff --git a/simulation/isaac-sim/docker/docker-compose.yaml b/simulation/isaac-sim/docker/docker-compose.yaml index dfd699aa4..efe146b86 100644 --- a/simulation/isaac-sim/docker/docker-compose.yaml +++ b/simulation/isaac-sim/docker/docker-compose.yaml @@ -8,8 +8,13 @@ services: dockerfile: docker/Dockerfile.isaac-ros tags: - *image_tag + # Floating tag republished by docker-build.yml on main/develop. The + # versioned tag above never exists yet on a PR (check-version-increment + # forces VERSION up), so it can only ever be a cache miss. + - &cache_tag ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_isaac-sim cache_from: - *image_tag + - *cache_tag container_name: isaac-sim entrypoint: "" command: > @@ -42,6 +47,15 @@ services: - NUM_ROBOTS=${NUM_ROBOTS:-1} - ENABLE_LIDAR=${ENABLE_LIDAR:-false} - ISAAC_SIM_HEADLESS=${ISAAC_SIM_HEADLESS:-false} + # Per-environment scene + spawn overrides (set by the mission runner per + # iteration from a mission's `environments:` entry; read by + # example_multi_drone_scene_import.py). Empty by default → the launch + # script keeps its hardcoded fallbacks. Without this passthrough the + # launch script never sees them. + - ENV_URL=${ENV_URL:-} + - STAGE_SCALE=${STAGE_SCALE:-} + - SPAWN_CONFIGS=${SPAWN_CONFIGS:-} + - SPAWN_HEIGHT_M=${SPAWN_HEIGHT_M:-} # Pegasus physics tuning — read by pegasus/simulator/params.py - PX4_PHYSICS_HZ=${PX4_PHYSICS_HZ:-100} - ARDUPILOT_PHYSICS_HZ=${ARDUPILOT_PHYSICS_HZ:-800} diff --git a/simulation/isaac-sim/extensions/PegasusSimulator b/simulation/isaac-sim/extensions/PegasusSimulator index 8c7a66409..c94a4fb1f 160000 --- a/simulation/isaac-sim/extensions/PegasusSimulator +++ b/simulation/isaac-sim/extensions/PegasusSimulator @@ -1 +1 @@ -Subproject commit 8c7a664095396bc9c4f4609531445d4cf6390fe4 +Subproject commit c94a4fb1f1a2abff03e28f6d42ee47ca705d10cb diff --git a/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py b/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py index 89fbe1865..d094d3cd8 100644 --- a/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py +++ b/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py @@ -12,6 +12,7 @@ "omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/5.1" ) +import json import os import sys import time @@ -50,7 +51,16 @@ #env/stage path and scale ENV_URL = f"omniverse://{NUCLEUS_SERVER}/Projects/AirStack/scenes/urban/allegheny_county_fire_academy/fire_academy.scene.usd" -STAGE_SCALE = 0.01 +# Per-environment overrides (the mission runner exports these per iteration +# from a mission's `environments:` entry — see osmo/missions/README.md); +# the hardcoded values above/below stay the standalone-run fallbacks. +# ENV_URL accepts a full URL or just the path after the nucleus server. +_env_url_override = os.environ.get("ENV_URL") +if _env_url_override: + ENV_URL = (_env_url_override if "://" in _env_url_override + else f"omniverse://{NUCLEUS_SERVER}/{_env_url_override.lstrip('/')}") + +STAGE_SCALE = float(os.environ.get("STAGE_SCALE") or 0.01) DRONE_USD = "~/.local/share/ov/data/documents/Kit/shared/exts/pegasus.simulator/pegasus/simulator/assets/Robots/Iris/iris.usd" @@ -73,13 +83,35 @@ # {"domain_id": 1, "x_m": 20.0, "y_m": -7.0, ...} # {"domain_id": 2, "x_m": 17.0, "y_m": 1.5, ...} -SPAWN_HEIGHT_ABOVE_FLOOR_M = 0.03 -DRONE_CONFIGS = [ - {"domain_id": 1, "x_m": 32.0, "y_m": 12.6, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": 0.75}, - {"domain_id": 2, "x_m": 28.0, "y_m": 14.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": 0.75}, - {"domain_id": 3, "x_m": 32.0, "y_m": 19.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": 0.75} +SPAWN_HEIGHT_ABOVE_FLOOR_M = float(os.environ.get("SPAWN_HEIGHT_M") or 0.03) +LIDAR_MIN_RANGE_M = 0.75 + +_DEFAULT_DRONE_CONFIGS = [ + {"domain_id": 1, "x_m": 32.0, "y_m": 12.6, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": LIDAR_MIN_RANGE_M}, + {"domain_id": 2, "x_m": 28.0, "y_m": 14.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": LIDAR_MIN_RANGE_M}, + {"domain_id": 3, "x_m": 32.0, "y_m": 19.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": LIDAR_MIN_RANGE_M} ] +# Spawn-location override: JSON list of dicts, one per drone. Only x_m / y_m +# are required — domain_id, z_m, orient, lidar_min_range default per drone. +# SPAWN_CONFIGS='[{"x_m": 32.0, "y_m": 12.6, "orient": [0, 0, -0.937, 0.35]}]' +_SPAWN_CONFIGS = os.environ.get("SPAWN_CONFIGS") +if _SPAWN_CONFIGS: + DRONE_CONFIGS = json.loads(_SPAWN_CONFIGS) + for _i, _c in enumerate(DRONE_CONFIGS, start=1): + _c.setdefault("domain_id", _i) + _c.setdefault("z_m", SPAWN_HEIGHT_ABOVE_FLOOR_M) + _c.setdefault("orient", [0.0, 0.0, 0.0, 1.0]) + _c.setdefault("lidar_min_range", LIDAR_MIN_RANGE_M) +else: + DRONE_CONFIGS = _DEFAULT_DRONE_CONFIGS + +# Logged so each iteration's chosen environment + layout is captured in the +# isaac-sim container logs, which mission_runner snapshots per iteration. +print(f"[spawn] ENV_URL={ENV_URL}", flush=True) +print(f"[spawn] STAGE_SCALE={STAGE_SCALE}", flush=True) +print(f"[spawn] DRONE_CONFIGS={json.dumps(DRONE_CONFIGS)}", flush=True) + # Top-down "map" camera. Captures one aerial of the static scene that the # GCS visualizer turns into a textured ground in Foxglove's 3D panel. The # camera centers on (OVERHEAD_CENTER_X_M, OVERHEAD_CENTER_Y_M) in world diff --git a/simulation/ms-airsim/docker/docker-compose.yaml b/simulation/ms-airsim/docker/docker-compose.yaml index 3a4035374..9b49c9098 100644 --- a/simulation/ms-airsim/docker/docker-compose.yaml +++ b/simulation/ms-airsim/docker/docker-compose.yaml @@ -8,8 +8,10 @@ services: dockerfile: Dockerfile tags: - *ms_airsim_image + - &ms_airsim_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_ms-airsim cache_from: - *ms_airsim_image + - *ms_airsim_cache container_name: ms-airsim entrypoint: "" command: /root/entrypoint.sh diff --git a/tests/README.md b/tests/README.md index 6aec42e1b..e9b9ea620 100644 --- a/tests/README.md +++ b/tests/README.md @@ -389,6 +389,11 @@ Regressions are flagged with :red_circle:, improvements with :green_circle:. ## CI/CD Integration +!!! note "Full pipeline guide" + For the end-to-end picture — architecture diagrams, job lifecycle, trigger + reference, what each mark catches, and how to fold CI into your development + loop — see **[CI/CD Pipeline on OSMO](../docs/development/intermediate/testing/ci_cd.md)**. + ### Workflow: `system-tests.yml` [`.github/workflows/system-tests.yml`](../../../../.github/workflows/system-tests.yml) runs on: @@ -409,7 +414,7 @@ Regressions are flagged with :red_circle:, improvements with :green_circle:. #### Jobs -**`run-tests`** runs on a freshly-spawned ephemeral OpenStack instance (`[self-hosted, airstack-ephemeral]`). The instance is provisioned per-job by the orchestrator described below and destroyed once the job completes. It installs dependencies, runs pytest, and uploads `tests/results/` as an artifact named `test-results--` with 90-day retention. +**`run-tests`** runs on a freshly-spawned ephemeral OSMO pod (`[self-hosted, airstack-ephemeral]`). The pod is submitted per-job by the orchestrator described below and destroyed once the job completes. It installs dependencies, runs pytest, and uploads `tests/results/` as an artifact named `test-results--` with 90-day retention. **`report`** runs on `ubuntu-latest` after `run-tests` (even if it failed). It: @@ -425,9 +430,9 @@ The workflow uses [`dawidd6/action-download-artifact@v6`](https://github.com/daw --- -## CI/CD Orchestrator (OpenStack-backed ephemeral runners) +## CI/CD Orchestrator (OSMO-backed ephemeral runners) -AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they execute on **truly ephemeral OpenStack instances** spawned per-job by an orchestrator. Each test job gets a fresh VM that is destroyed once the job completes — no Docker layer carryover, no leaked containers, no shared host state. +AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they execute on **truly ephemeral [NVIDIA OSMO](https://nvidia.github.io/OSMO/) pods** submitted per-job by an orchestrator. Each test job gets a fresh GPU pod that is destroyed once the job completes — no Docker layer carryover, no leaked containers, no shared host state. (This replaced an OpenStack-Nova backend; the GitHub side and the per-job-destroy model are unchanged.) ### Architecture @@ -436,19 +441,19 @@ AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they │ Orchestrator VM (airstack-ci-cd-orchestrator) │ │ • polls GitHub for queued workflow_jobs │ │ • mints single-use JIT runner tokens │ -│ • spawns / reaps ephemeral instances via OpenStack Nova │ -│ • holds the GitHub PAT and OpenStack application credential│ +│ • submits / reaps ephemeral OSMO workflows via osmo CLI │ +│ • holds the GitHub PAT and OSMO service-account token │ └────────────┬───────────────────────────────────┬─────────────┘ │ │ ▼ ▼ ┌──────────────────────────────┐ ┌────────────────────────────────┐ │ Ephemeral worker (per job) │ │ GitHub Actions queue │ -│ Image: Ubuntu-24.04-GPU- │ │ workflow_job status=queued │ -│ Headless │ │ labels: [self-hosted, │ -│ cloud-init bootstraps Docker │ │ airstack-ephemeral] │ -│ + nvidia-container-toolkit + │ └────────────────────────────────┘ -│ GH Actions runner; runs ONE │ -│ job, then is destroyed. │ +│ Prebaked airstack-ci-runner │ │ workflow_job status=queued │ +│ image: Docker + nvidia CTK + │ │ labels: [self-hosted, │ +│ GH runner. Privileged pod │ │ airstack-ephemeral] │ +│ starts dockerd, runs ONE │ └────────────────────────────────┘ +│ job (JIT), then the pod │ +│ is destroyed. │ └──────────────────────────────┘ ``` @@ -456,21 +461,22 @@ AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they | Concern | Mitigation | |---------|------------| -| Cross-job state pollution (Docker cache, dangling networks, leftover artifacts) | Each job runs on a fresh VM. Spent VM is destroyed within ~30 s of job completion. | +| Cross-job state pollution (Docker cache, dangling networks, leftover artifacts) | Each job runs on a fresh OSMO pod, destroyed within ~30 s of job completion. | | Fork PRs executing arbitrary code | Workflow's `if: github.event.pull_request.head.repo.full_name == github.repository` — fork PRs skipped. | -| Runner running as root | The runner runs as the unprivileged `ubuntu` user inside an instance whose only purpose is one job. | -| Docker socket gives root-equivalent access | Bounded to a single one-shot VM. The orchestrator host doesn't expose Docker at all. | +| Runner runs privileged (root) for docker-in-docker | The pod is privileged (needed to run `airstack up`/compose), but it is single-use, scoped to the dedicated CI pool, and only same-repo code ever reaches it. | +| Docker socket gives root-equivalent access | Bounded to a single one-shot pod. The orchestrator host doesn't expose Docker at all. | | Long-lived PAT on the runner host | The PAT lives only on the orchestrator. Workers receive a single-use **JIT runner config** — a base64 token bound to one runner registration. | -| Persistent OpenStack creds tied to a user password | Orchestrator authenticates with an **application credential** (revocable, scoped) instead of `openrc.sh`. | +| Persistent creds tied to a personal account | Orchestrator authenticates with a shared, non-personal **OSMO service-account token** (revocable, scoped to the CI pool), not an individual's login. | ### Setup -The orchestrator service code, cloud-init template, systemd unit, and full setup runbook live in [`.github/orchestrator/`](../../../../.github/orchestrator/). See [`.github/orchestrator/README.md`](ci-cd-orchestrator.md) for: +The orchestrator service code, OSMO runner-workflow template, runner image, systemd unit, and full setup runbook live in [`.github/orchestrator/`](../../../../.github/orchestrator/). See [`.github/orchestrator/README.md`](ci-cd-orchestrator.md) for: -- creating the OpenStack application credential and `clouds.yaml` -- staging the GitHub PAT -- running `setup.sh` on the orchestrator VM -- filling in flavor / network / keypair / security-group in `/etc/airstack-orchestrator/config.yaml` +- obtaining the OSMO service-account token and a dedicated CI GPU pool (with privileged mode enabled) +- building and pushing the runner image (`runner.Dockerfile`) +- staging the GitHub PAT and the OSMO token +- running `setup.sh` on the orchestrator host (installs the `osmo` CLI) +- filling in osmo_url / pool / platform / runner_image / resources in `/etc/airstack-orchestrator/config.yaml` - enabling and verifying the `airstack-orchestrator.service` systemd unit ### Runner labels diff --git a/tests/colcon_unit_test_packages.yaml b/tests/colcon_unit_test_packages.yaml index 5e0bd0ebb..07e06a5b4 100644 --- a/tests/colcon_unit_test_packages.yaml +++ b/tests/colcon_unit_test_packages.yaml @@ -9,5 +9,7 @@ robot: packages: - natnet_ros2 - lidar_point_cloud_filter - # Skips ament_copyright / flake8 / pep257 on Python packages; run linters separately. - pytest_args: "-m not linter" + # Linter skip lives in lidar_point_cloud_filter setup.cfg + test/conftest.py. + # ament pytest does not honor PYTEST_ADDOPTS -m. + # launch_testing is skipped via PYTEST_DISABLE_PLUGIN_AUTOLOAD in the test. + pytest_args: [] diff --git a/tests/conftest.py b/tests/conftest.py index 2a51c569a..0edccf716 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,20 +69,42 @@ def load_colcon_unit_test_config(workspace="robot"): raise ValueError( f"'{workspace}.packages' is empty in {COLCON_UNIT_TEST_PACKAGES_YAML.name}" ) - return packages, cfg.get("pytest_args", "") + raw_args = cfg.get("pytest_args", []) + if isinstance(raw_args, str): + pytest_args = shlex.split(raw_args) if raw_args else [] + elif isinstance(raw_args, list): + pytest_args = [str(a) for a in raw_args] + else: + raise TypeError( + f"'{workspace}.pytest_args' must be a list or string in " + f"{COLCON_UNIT_TEST_PACKAGES_YAML.name}, got {type(raw_args).__name__}" + ) + return packages, pytest_args def colcon_test_robot_command(workspace="robot"): - """Shell command for colcon test over unit-test packages (robot workspace).""" - packages, pytest_args = load_colcon_unit_test_config(workspace) + """Shell command for colcon test over unit-test packages (robot workspace). + + Pytest flags from the YAML are *not* put on this command. colcon's + ``--pytest-args`` is a single nargs='*' option (last occurrence wins), + and nesting those tokens through ``bash -ic`` also breaks quoting. + Pass them as ``PYTEST_ADDOPTS`` via ``docker_exec(..., env=...)``. + """ + packages, _ = load_colcon_unit_test_config(workspace) pkg_list = " ".join(packages) - cmd = ( + return ( f"colcon test --packages-select {pkg_list} " "--event-handlers console_direct+ --return-code-on-test-failure" ) - if pytest_args: - cmd += f' --pytest-args "{pytest_args}"' - return cmd + + +def format_pytest_addopts(pytest_args): + """Build a PYTEST_ADDOPTS value that pytest will shlex-split back to tokens. + + Do not name this pytest_*: conftest functions with that prefix are treated + as pytest hooks and fail collection (exit code 3). + """ + return " ".join(shlex.quote(a) for a in pytest_args) # Unit tests live co-located with their ROS 2 packages in robot/ros_ws/src/. # Thin proxy files under tests/robot/ re-export those tests so that # `pytest tests/` and `airstack test -m unit` discover them without any @@ -124,6 +146,9 @@ def pytest_addoption(parser): parser.addoption("--trajectory-types", default="Circle,Figure8,Racetrack,Line", help="Comma-separated fixed trajectory types to sweep in " "test_fixed_trajectory. Default: Circle,Figure8,Racetrack,Line") + parser.addoption("--no-image-build", action="store_true", default=False, + help="CI flag: skip image-build in system-tests.yml. " + "Ignored by pytest itself.") def pytest_configure(config): @@ -343,8 +368,13 @@ def _run_teed(cmd_list, timeout, log_name=None, env=None, cwd=None): return result -def docker_exec(container, cmd, timeout=60, log_name=None): - full_cmd = ["docker", "exec", container, "bash", "-c", cmd] +def docker_exec(container, cmd, timeout=60, log_name=None, env=None): + """Run ``cmd`` in ``container``. ``env`` is passed as ``docker exec -e``.""" + full_cmd = ["docker", "exec"] + if env: + for key, value in env.items(): + full_cmd.extend(["-e", f"{key}={value}"]) + full_cmd.extend([container, "bash", "-c", cmd]) return _run_teed(full_cmd, timeout=timeout, log_name=log_name) diff --git a/tests/system/test_build_packages.py b/tests/system/test_build_packages.py index 5c54cfee3..aa84b0464 100644 --- a/tests/system/test_build_packages.py +++ b/tests/system/test_build_packages.py @@ -1,10 +1,12 @@ +import shlex from pathlib import Path import pytest from conftest import (AIRSTACK_ROOT, airstack_cmd, colcon_test_robot_command, - docker_exec, load_colcon_unit_test_config, logger, - read_log_tail, wait_for_container) + docker_exec, format_pytest_addopts, + load_colcon_unit_test_config, logger, read_log_tail, + wait_for_container) def _warn_if_prebuilt(*ws_paths): @@ -52,7 +54,7 @@ def test_colcon_test_robot(self): Package list and pytest args come from tests/colcon_unit_test_packages.yaml. Workspace-wide ament linter tests are not gated here. """ - packages, _ = load_colcon_unit_test_config("robot") + packages, pytest_args = load_colcon_unit_test_config("robot") try: result = airstack_cmd("up", "robot-desktop", env_overrides={"AUTOLAUNCH": "false", "DISPLAY": ""}, @@ -69,10 +71,18 @@ def test_colcon_test_robot(self): ) assert build.returncode == 0, f"colcon build (with testing) failed:\n{read_log_tail()}" + # PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 skips launch_testing before + # pytest 8.1+ validates its removed path= hook. Linter skip is in + # the package setup.cfg / test/conftest.py (ament pytest ignores + # PYTEST_ADDOPTS -m). + exec_env = {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"} + if pytest_args: + exec_env["PYTEST_ADDOPTS"] = format_pytest_addopts(pytest_args) test = docker_exec( container, - f"bash -ic '{colcon_test_robot_command('robot')}'", + f"bash -ic {shlex.quote(colcon_test_robot_command('robot'))}", timeout=300, + env=exec_env, ) assert test.returncode == 0, ( f"colcon test failed (packages: {', '.join(packages)}):\n{read_log_tail()}"