From 4bd534aa637a8866dd0ea69710af230c1044b4ad Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Wed, 10 Jun 2026 15:52:26 -0400 Subject: [PATCH 01/27] added mission_runner.py to automate running missions in osmo --- .airstack/modules/osmo.sh | 198 +++++++ .env | 2 +- .gitignore | 3 + docs/tutorials/airstack_on_osmo.md | 42 +- osmo/README.md | 59 +- osmo/missions/README.md | 146 +++++ osmo/missions/example_takeoff_land.yaml | 68 +++ osmo/workflows/airstack-mission.yaml | 92 ++++ osmo/workspace/Dockerfile | 1 + osmo/workspace/entrypoint.sh | 78 ++- osmo/workspace/mission_runner.py | 684 ++++++++++++++++++++++++ 11 files changed, 1344 insertions(+), 29 deletions(-) create mode 100644 osmo/missions/README.md create mode 100644 osmo/missions/example_takeoff_land.yaml create mode 100644 osmo/workflows/airstack-mission.yaml create mode 100644 osmo/workspace/mission_runner.py diff --git a/.airstack/modules/osmo.sh b/.airstack/modules/osmo.sh index 053decbee..a10a47809 100755 --- a/.airstack/modules/osmo.sh +++ b/.airstack/modules/osmo.sh @@ -686,6 +686,199 @@ function cmd_osmo_foxglove { --connect-timeout "$OSMO_PF_TIMEOUT" } +# osmo:mission — submit airstack-mission.yaml with a mission spec selected. +# +# Usage: airstack osmo:mission [--pool POOL] [--key PATH] +# [--branch BRANCH] [--no-keep-alive] +# +# is a repo-relative path (e.g. osmo/missions/example_takeoff_land.yaml). +# The pod clones the branch and runs the mission spec from that clone, so the +# mission file must be committed and pushed. --no-keep-alive makes the task +# exit when the mission ends (frees the GPU, triggers the workflow's +# `outputs:` upload) instead of sleeping for `airstack osmo:fetch`. +function cmd_osmo_mission { + _osmo_check_cli || return 1 + + local mission="" + local pool="${OSMO_POOL:-}" + local pubkey_file="" + local branch="" + local branch_explicit=false + local keep_alive="true" + local extra_args=() + + while [ $# -gt 0 ]; do + case "$1" in + --pool) pool="$2"; shift 2 ;; + --key) pubkey_file="$2"; shift 2 ;; + --branch) branch="$2"; branch_explicit=true; shift 2 ;; + --no-keep-alive) keep_alive="false"; shift ;; + -*) extra_args+=("$1"); shift ;; + *) + if [ -z "$mission" ]; then mission="$1"; else extra_args+=("$1"); fi + shift ;; + esac + done + + if [ -z "$mission" ]; then + log_error "Usage: airstack osmo:mission [--pool POOL] [--branch BRANCH] [--no-keep-alive]" + log_error "Available missions:" + ls "${PROJECT_ROOT}/osmo/missions/"*.yaml 2>/dev/null \ + | sed "s|${PROJECT_ROOT}/| |" >&2 + return 1 + fi + # Normalize to a repo-relative path — that's what the pod resolves + # against its clone of the branch. + mission="${mission#"${PROJECT_ROOT}"/}" + if [ ! -f "${PROJECT_ROOT}/${mission}" ]; then + log_error "Mission file not found locally: ${PROJECT_ROOT}/${mission}" + return 1 + fi + + if [ -z "$pubkey_file" ]; then + if ! pubkey_file="$(_osmo_pick_pubkey)"; then + log_error "No SSH public key found in ~/.ssh. Generate one with: ssh-keygen -t ed25519" + return 1 + fi + fi + + local workflow_yaml="${PROJECT_ROOT}/osmo/workflows/airstack-mission.yaml" + if [ ! -f "$workflow_yaml" ]; then + log_error "Workflow file not found: ${workflow_yaml}" + return 1 + fi + + # The pod runs the mission file from its clone of origin/, so an + # unpushed mission spec is the most common "why is it running the wrong + # thing" failure — same auto-pin + pushed check as osmo:up. + if [ "$branch_explicit" = false ] && [ -z "$branch" ]; then + branch="$(_osmo_local_branch)" + if [ -n "$branch" ]; then + log_info "Auto-detected local branch '${branch}'; pod will clone from origin/${branch} (override with --branch main)." + fi + fi + if [ -n "$branch" ]; then + _osmo_check_branch_pushed "$branch" + fi + + local cmd=(osmo workflow submit "$workflow_yaml") + if [ -n "$pool" ]; then + cmd+=(--pool "$pool") + else + log_warn "No --pool provided and OSMO_POOL is unset; using your osmo profile's default pool." + fi + # Single --set-env: the flag is variadic and a second occurrence silently + # drops the first (see cmd_osmo_up). + local env_kvs=( + "SSH_PUB_KEY=$(cat "$pubkey_file")" + "OSMO_MISSION_FILE=${mission}" + "OSMO_MISSION_KEEP_ALIVE=${keep_alive}" + ) + if [ -n "$branch" ]; then + env_kvs+=("AIRSTACK_BRANCH=${branch}") + fi + cmd+=(--set-env "${env_kvs[@]}") + if [ ${#extra_args[@]} -gt 0 ]; then + cmd+=("${extra_args[@]}") + fi + + log_info "Submitting mission '${mission}' (keep_alive=${keep_alive}): ${cmd[*]}" + local output + if ! output="$("${cmd[@]}" 2>&1)"; then + echo "$output" >&2 + log_error "osmo workflow submit failed." + return 1 + fi + echo "$output" + + local wf_id + wf_id="$(echo "$output" | awk -F'- ' '/^Workflow ID/ {print $2; exit}' | tr -d ' \r\n')" + if [ -z "$wf_id" ]; then + log_warn "Could not parse workflow id from submit output. Set it manually:" + log_warn " echo > ${OSMO_STATE_FILE}" + return 0 + fi + _osmo_save_wf_id "$wf_id" + + log_info "Next steps:" + log_info " airstack osmo:logs # follow mission progress" + log_info " airstack osmo:fetch # download bags + results (keep-alive mode)" + log_info " airstack osmo:down # cancel when done (results die with the pod!)" +} + +# osmo:fetch — download mission results (mcap bags, logs, summaries) from the +# pod to the laptop over the authenticated ssh port-forward. +# +# Usage: airstack osmo:fetch [dest-dir] +# +# Incremental and resumable (rsync): safe to run mid-mission to pull finished +# iterations while the next one flies, and again later to top up. Falls back +# to scp -r (non-incremental) if rsync isn't installed. +# +# Note: the osmo CLI also ships `osmo workflow rsync download`, which could +# replace the port-forward + ssh below; we use the ssh path because the +# sshd + port-forward channel is already validated infrastructure for this +# workflow (osmo:ide) and works uniformly across osmo CLI versions. +function cmd_osmo_fetch { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + local dest="${1:-./osmo-results}" + local remote_path="/root/AirStack/osmo/results/" # trailing slash: rsync + # follows the symlink to + # /osmo/output/... on pods + local local_port="${OSMO_SSH_PORT%%:*}" + local ssh_opts=(-p "$local_port" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR) + + # Reuse an existing ssh port-forward (e.g. from osmo:ide) or spawn one + # for the duration of the fetch — same pattern as osmo:ide. + local pf_pid="" + if ! nc -z localhost "$local_port" 2>/dev/null; then + log_info "osmo workflow port-forward ${wf} workspace --port ${OSMO_SSH_PORT} (for the duration of the fetch)" + osmo workflow port-forward "$wf" workspace --port "$OSMO_SSH_PORT" --connect-timeout 600 \ + > "${OSMO_STATE_DIR}/fetch-pf.log" 2>&1 & + pf_pid=$! + trap '[ -n "'"$pf_pid"'" ] && kill "'"$pf_pid"'" 2>/dev/null; trap - EXIT INT TERM' EXIT INT TERM + local waited=0 + until nc -z localhost "$local_port" 2>/dev/null; do + sleep 1; waited=$((waited+1)) + if [ "$waited" -ge 30 ]; then + log_error "Timed out waiting for port-forward on :${local_port}. Log: ${OSMO_STATE_DIR}/fetch-pf.log" + return 1 + fi + if ! kill -0 "$pf_pid" 2>/dev/null; then + log_error "port-forward exited early. Tail:" + tail -10 "${OSMO_STATE_DIR}/fetch-pf.log" >&2 + return 1 + fi + done + fi + + mkdir -p "$dest" + log_info "Fetching ${remote_path} → ${dest}" + local rc + if command -v rsync >/dev/null 2>&1; then + rsync -az --partial --info=progress2 -e "ssh ${ssh_opts[*]}" \ + "root@localhost:${remote_path}" "$dest/" + rc=$? + else + log_warn "rsync not found; falling back to scp -r (non-incremental)." + scp "${ssh_opts[@]}" -r "root@localhost:${remote_path}." "$dest/" + rc=$? + fi + + if [ -n "$pf_pid" ]; then + kill "$pf_pid" 2>/dev/null + trap - EXIT INT TERM + fi + + if [ "$rc" -ne 0 ]; then + log_error "Fetch failed (exit ${rc}). Is the workflow still running, and has the mission produced results yet?" + return 1 + fi + log_info "Done. Open any .mcap under ${dest} directly in Foxglove (Open local file)." +} + # osmo:down — cancel the active workflow. Reminds you to push first. function cmd_osmo_down { _osmo_check_cli || return 1 @@ -693,6 +886,7 @@ function cmd_osmo_down { log_warn "About to cancel workflow '${wf}'." log_warn "Anything not pushed to git in /root/AirStack inside the pod will be LOST." + log_warn "Mission results (bags/logs) on the pod are lost too — run 'airstack osmo:fetch' first." log_warn "Hit Ctrl-C in the next 5 seconds to abort." sleep 5 osmo workflow cancel "$wf" @@ -703,6 +897,8 @@ function cmd_osmo_down { function register_osmo_commands { COMMANDS["osmo:setup"]="cmd_osmo_setup" COMMANDS["osmo:up"]="cmd_osmo_up" + COMMANDS["osmo:mission"]="cmd_osmo_mission" + COMMANDS["osmo:fetch"]="cmd_osmo_fetch" COMMANDS["osmo:logs"]="cmd_osmo_logs" COMMANDS["osmo:ide"]="cmd_osmo_ide" COMMANDS["osmo:webrtc"]="cmd_osmo_webrtc" @@ -711,6 +907,8 @@ function register_osmo_commands { COMMAND_HELP["osmo:setup"]="One-time per-user OSMO credential setup (airlab-docker-registry, airlab-docker-login, airlab-nucleus)" COMMAND_HELP["osmo:up"]="Submit osmo/workflows/airstack-dev.yaml with your SSH pubkey injected (--pool POOL, --key PATH, --branch BRANCH)" + COMMAND_HELP["osmo:mission"]="Submit a batch mission (osmo/missions/*.yaml): repeated up→fly→record→down cycles (--pool POOL, --branch BRANCH, --no-keep-alive)" + COMMAND_HELP["osmo:fetch"]="Download mission results (mcap bags, logs, summaries) from the pod over ssh — incremental, safe to run mid-mission (osmo:fetch [dest-dir])" COMMAND_HELP["osmo:logs"]="Follow the workspace task logs (osmo workflow logs -t workspace -n 500; OSMO_LOGS_TASK / OSMO_LOGS_TAIL override)" COMMAND_HELP["osmo:ide"]="Port-forward sshd (2200:22) and open VS Code/Cursor on Host airstack-osmo" COMMAND_HELP["osmo:webrtc"]="Port-forward Isaac Sim WebRTC ranges (TCP foreground + UDP background)" diff --git a/.env b/.env index 82cc01ccb..412801cb6 100644 --- a/.env +++ b/.env @@ -12,7 +12,7 @@ PROJECT_NAME="airstack" # If you've run ./airstack.sh setup, then this will auto-generate from the git commit hash every time a change is made # to a Dockerfile or docker-compose.yaml file. Otherwise this can also be set explicitly to make a release version. # auto-generated from git commit hash -VERSION="0.19.0-alpha.3" +VERSION="8b927e46" # Choose "dev" or "prebuilt". "dev" is for mounted code that must be built live. "prebuilt" is for built ros_ws baked into the image DOCKER_IMAGE_BUILD_MODE="dev" # Where to push and pull images from. Can replace with your docker hub username if using docker hub. diff --git a/.gitignore b/.gitignore index 4868b5c74..d14138ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,9 @@ simulation/ms-airsim/assets/scenes/* # Test results tests/results/ +# OSMO mission results (local runs of osmo/workspace/mission_runner.py) +osmo/results/ + # Local-only — embedded sibling repo, not part of this branch common/rayfronts/ diff --git a/docs/tutorials/airstack_on_osmo.md b/docs/tutorials/airstack_on_osmo.md index e9cfa8974..9f093e8c0 100644 --- a/docs/tutorials/airstack_on_osmo.md +++ b/docs/tutorials/airstack_on_osmo.md @@ -547,6 +547,45 @@ osmo workflow cancel $WF +## Batch missions (unattended runs) + +Everything above is the *interactive* workflow. The same pod can instead run +**missions**: declarative YAML files (in +[`osmo/missions/`](https://github.com/castacks/AirStack/blob/main/osmo/missions/)) +that script repeated cycles of bring-up → fly → record → tear-down with no +human attached. Each iteration restarts the containers, records mcap bag +files (Foxglove's native format — open the `.mcap` directly, no conversion), +and snapshots container logs and per-step results. + +```bash +# Submit a mission (auto-pins your current branch, like osmo:up): +./airstack.sh osmo:mission osmo/missions/example_takeoff_land.yaml --pool airstack + +# Watch it fly: +./airstack.sh osmo:logs + +# Pull bags + logs + summaries to your laptop — incremental, safe to run +# mid-mission and again later to top up: +./airstack.sh osmo:fetch ./results/ + +# When you have everything (results die with the pod!): +./airstack.sh osmo:down +``` + +A mission step can be any robot task action (`takeoff`, `land`, `navigate`, +`semantic_search`, `exploration`, `coverage`, …), a timed wait, a topic pub, +a service call, or an arbitrary `ros2`/shell command. Spec schema and step +reference: +[`osmo/missions/README.md`](https://github.com/castacks/AirStack/blob/main/osmo/missions/README.md). + +By default the pod **stays alive after the mission ends** so you can +`osmo:fetch` whenever you're ready (mind the workflow's 24h `exec_timeout`). +For fire-and-forget batches, submit with `--no-keep-alive`: the pod exits +cleanly when the mission ends, freeing the GPU — and uploading the results +directory to object storage automatically if the workflow's `outputs:` +block is configured (lab-admin setup; see +[`osmo/README.md`](https://github.com/castacks/AirStack/blob/main/osmo/README.md)). + ## Troubleshooting | Symptom | Likely cause | Fix | @@ -572,7 +611,8 @@ osmo workflow cancel $WF | Uncommitted edits in the IDE | Pod-local working tree | **No** | | `colcon build` outputs (`build/`, `install/`, `log/`) | `/root/AirStack/**/ros_ws/...` | **No** (gitignored Linux x86_64 binaries; rebuild trivially) | | Inner-dockerd image cache | Pod-local Docker layer cache | **No** | -| Bag files, sim recordings, debug screenshots | `/root/AirStack/bags/`, etc. | **No** — pull selectively via `osmo workflow rsync download "$(cat ~/.airstack/osmo-state)" :` *before* tearing down | +| Mission results (mcap bags, logs, summaries) | `/root/AirStack/osmo/results/` | **No** — run `./airstack.sh osmo:fetch` *before* tearing down | +| Other bag files, sim recordings, debug screenshots | `/root/AirStack/bags/`, etc. | **No** — pull selectively via `osmo workflow rsync download "$(cat ~/.airstack/osmo-state)" :` *before* tearing down | The rule of thumb: **commit + push every time you'd save a file in a git-tracked sense.** The Source Control panel is the persistence boundary. diff --git a/osmo/README.md b/osmo/README.md index 91b41dbd5..410ed21c2 100644 --- a/osmo/README.md +++ b/osmo/README.md @@ -7,11 +7,17 @@ through [NVIDIA OSMO](https://github.com/NVIDIA/OSMO): osmo/ ├── README.md # This file (admin / operator reference) ├── workflows/ -│ └── airstack-dev.yaml # The OSMO workflow students submit +│ ├── airstack-dev.yaml # Interactive dev workflow (IDE over Remote-SSH) +│ └── airstack-mission.yaml # Batch mission workflow (unattended flights) +├── missions/ +│ ├── README.md # Mission spec schema reference +│ └── example_takeoff_land.yaml # Reference mission: takeoff → hover → land ×3 └── workspace/ ├── Dockerfile # The airstack-osmo-workspace image ├── sshd_config # Pubkey-only sshd config baked into the image - └── entrypoint.sh # Pod startup: sshd, dockerd, clone, airstack up + ├── entrypoint.sh # Pod startup: sshd, dockerd, clone, then + │ # dev mode (airstack up) or mission mode + └── mission_runner.py # Batch executor (run from the clone, not the image) ``` The student-facing walkthrough lives in @@ -21,9 +27,9 @@ README is the **lab admin / operator** reference: pool requirements, workspace image build & push, validation stages, plus a credential summary for context. -> **Scope:** developer workflow only. CI/CD on OSMO is **not** part of this -> integration — the existing `system-tests.yml` + OpenStack orchestrator path -> is unchanged. +> **Scope:** developer workflow + batch missions. CI/CD on OSMO is **not** +> part of this integration — the existing `system-tests.yml` + OpenStack +> orchestrator path is unchanged. ## Architecture in one minute @@ -47,6 +53,49 @@ app.foxglove.dev ── ws ────► port-forward 8766 ────► airstack.sh up brings these 3 up ``` +## Mission mode (batch runs) + +`airstack-mission.yaml` reuses the same workspace image and DinD pod, but +instead of one interactive `airstack up`, the entrypoint hands off to +[`workspace/mission_runner.py`](workspace/mission_runner.py), which executes +a declarative mission spec from [`missions/`](missions/) — repeated cycles of: + +``` +airstack down → airstack up → wait for PX4 ready → record mcap bags +→ run steps (takeoff / land / navigate / semantic search / any ros2 command) +→ collect bags + container logs → airstack down +``` + +Submit, monitor, and download: + +```bash +airstack osmo:mission osmo/missions/example_takeoff_land.yaml --pool +airstack osmo:logs # follow mission progress +airstack osmo:fetch ./results/ # rsync bags/logs/summaries to the laptop +airstack osmo:down # cancel (fetch first — results die with the pod) +``` + +Key behaviors: + +- **The mission spec and runner come from the clone**, not the image — what + you push on your branch is what runs. The workspace image only needs a + rebuild when `Dockerfile`, `sshd_config`, or `entrypoint.sh` change. +- **Bags are mcap** (`ros2 bag record -s mcap`) — open the `.mcap` files + directly in Foxglove, no conversion or local ROS install. +- **Results location:** `/osmo/output/airstack-mission-results///` + with a symlink at `/root/AirStack/osmo/results` (the path `osmo:fetch` + pulls). Artifacts are collected even for failed iterations. +- **`OSMO_MISSION_KEEP_ALIVE`** (default `true`): the pod sleeps after the + mission so you can fetch over ssh. Set `false` (or submit with + `osmo:mission --no-keep-alive`) for fire-and-forget: the task exits + cleanly when the mission ends, freeing the GPU — and if the workflow's + `outputs:` block is configured with a destination bucket, OSMO uploads + `/osmo/output` automatically on that exit. A **canceled** workflow does + not upload outputs, so in keep-alive mode `osmo:fetch` is the retrieval + path. + +Mission spec schema and step types: [`missions/README.md`](missions/README.md). + ## Pool requirements The OSMO pool the workflow runs on must satisfy: diff --git a/osmo/missions/README.md b/osmo/missions/README.md new file mode 100644 index 000000000..d0d95823d --- /dev/null +++ b/osmo/missions/README.md @@ -0,0 +1,146 @@ +# AirStack mission specs + +A **mission** is a declarative YAML file executed by +[`osmo/workspace/mission_runner.py`](../workspace/mission_runner.py). Each +mission runs one or more full iterations of: + +``` +airstack down → airstack up → wait for PX4 ready → start mcap recording +→ run steps → stop recording → collect bags + container logs → airstack down +``` + +Missions live in this directory so they're versioned with the code they +exercise — the OSMO pod clones your branch, so whatever you push is what runs. + +Submit to OSMO with: + +```bash +airstack osmo:mission osmo/missions/.yaml --pool +``` + +Or run locally on any machine that can `airstack up` (no OSMO involved): + +```bash +python3 osmo/workspace/mission_runner.py osmo/missions/.yaml \ + --airstack-root "$(pwd)" +``` + +`--dry-run` validates the spec and prints the merged config without touching +Docker. + +## Results + +``` +osmo/results/// +├── summary.json # per-iteration status + durations +└── iter_001/ + ├── bags/robot_1/*.mcap # open directly in Foxglove (no conversion) + ├── logs/.log # docker logs snapshot per container + ├── ready.json # per-robot seconds-to-PX4-ready + ├── steps.json # per-step command, output tail, pass/fail + └── iteration.json # iteration summary +``` + +On an OSMO pod the actual storage is `/osmo/output/airstack-mission-results` +(with `osmo/results` symlinked to it), so a workflow `outputs:` block uploads +everything automatically when the task exits. Download from your laptop at +any time while the pod is alive with `airstack osmo:fetch [dest]`. + +Artifacts are collected **even when an iteration fails** — a failed flight's +bag is usually the most interesting one. + +## Schema + +Top-level keys (everything except `steps` is optional): + +| Key | Default | Meaning | +|---|---|---| +| `name` | filename stem | Results directory name | +| `env` | `{}` | Env vars exported before each `airstack up` (`NUM_ROBOTS`, `COMPOSE_PROFILES`, `ISAAC_SIM_SCRIPT_NAME`, …) | +| `iterations` | `1` | Number of full up→fly→down cycles | +| `ready.timeout_s` | `600` | Max seconds to wait for PX4 readiness per iteration | +| `ready.poll_interval_s` | `5` | Seconds between readiness polls | +| `record.enabled` | `true` | Record an mcap per robot per iteration | +| `record.topics` | tf + odom set | Topics to record; `{robot}` → `robot_N` | +| `record.all` | `false` | Record **all** topics (`ros2 bag record -a`) — large | +| `on_step_failure` | `abort_iteration` | `continue` \| `abort_iteration` \| `abort_mission` | +| `up_timeout_s` | `3600` | `airstack up` timeout (first up on a fresh pod pulls images) | +| `down_timeout_s` | `300` | `airstack down` timeout | +| `robot_setup_bash` | robot ws `setup.bash` | Workspace sourced before `ros2` commands | +| `steps` | — | Ordered list of steps (below) | + +### Steps + +Every step type accepts the placeholders `{robot}` → `robot_N` and `{n}` → `N` +in its strings, and runs once per robot in `robots:` (`all` by default, or a +list like `[1, 3]`). + +**`action`** — send a goal to a robot task action server and wait for the +result. The step passes when the action result reports `success: true`. + +```yaml +- action: + task: takeoff # → /robot_N/tasks/ + goal: {target_altitude_m: 10.0, velocity_m_s: 1.0} + timeout_s: 120 # default 120 + robots: all + # type: task_msgs/action/TakeoffTask # derived from task name if omitted +``` + +Available tasks (action type is derived as `task_msgs/action/Task`): +`takeoff`, `land`, `fixed_trajectory`, `navigate`, `exploration`, `coverage`, +`semantic_search`, `chat`. Goal fields are defined in +[`common/ros_packages/msgs/task_msgs/action/`](../../common/ros_packages/msgs/task_msgs/action/). +Multi-robot action goals are sent **in parallel** across robots. + +**`wait`** — sleep for N seconds (e.g. hover, let a planner run): + +```yaml +- wait: 30 +``` + +**`run`** — arbitrary command; the escape hatch that makes any ROS 2 command +work without runner changes. The step fails on non-zero exit unless +`expect_success: false`. + +```yaml +- run: + container: robot_1 # robot_N → exec in the robot container on robot + # N's DDS domain (ros2 is sourced for you); + # pod → run on the pod itself (cwd = AirStack root); + # any other value → literal container name, domain 0 + cmd: ros2 topic echo --once /{robot}/odometry + timeout_s: 60 + expect_success: true +``` + +**`topic_pub`** — `ros2 topic pub --once` per robot: + +```yaml +- topic_pub: + topic: /{robot}/some_input + type: std_msgs/msg/Bool + msg: {data: true} +``` + +**`service_call`** — `ros2 service call` per robot: + +```yaml +- service_call: + service: /{robot}/some_service + type: std_srvs/srv/Trigger + request: {} +``` + +## Notes + +- For `NUM_ROBOTS > 1` on Isaac Sim, set + `ISAAC_SIM_SCRIPT_NAME: example_multi_px4_pegasus_launch_script.py` in + `env:` — the default script spawns a single drone. +- Missions run unattended: keep `ISAAC_SIM_HEADLESS: "true"`, + `ISAAC_SIM_USE_STANDALONE: "true"` and `PLAY_SIM_ON_START: "true"` unless + you're watching via the WebRTC livestream profile. +- Recording camera/LiDAR topics (or `record.all: true`) is the bag-size + driver — budget pod `storage:` accordingly. +- `/tf` and `/tf_static` are in the default topic set because without them a + Foxglove 3D panel can't pose anything during replay. diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml new file mode 100644 index 000000000..537e95b37 --- /dev/null +++ b/osmo/missions/example_takeoff_land.yaml @@ -0,0 +1,68 @@ +# Example mission: takeoff → hover 30s → land, 3 times, Isaac Sim headless. +# +# Run on OSMO: +# airstack osmo:mission osmo/missions/example_takeoff_land.yaml --pool +# +# Or locally against a machine that can run `airstack up` (no OSMO needed): +# python3 osmo/workspace/mission_runner.py osmo/missions/example_takeoff_land.yaml \ +# --airstack-root "$(pwd)" +# +# Schema reference: osmo/missions/README.md + +name: example_takeoff_land + +# Exported before every `airstack up`. Anything the compose stack reads from +# the environment can go here (.env values, ISAAC_SIM_SCRIPT_NAME, etc.). +env: + NUM_ROBOTS: 1 + COMPOSE_PROFILES: desktop,isaac-sim + ISAAC_SIM_HEADLESS: "true" + ISAAC_SIM_USE_STANDALONE: "true" + # The default script only spawns one drone; for NUM_ROBOTS > 1 switch to + # example_multi_px4_pegasus_launch_script.py. + ISAAC_SIM_SCRIPT_NAME: example_one_px4_pegasus_launch_script.py + # Missions are unattended — the sim must start playing without a GUI click. + PLAY_SIM_ON_START: "true" + +# Full up → fly → collect → down cycles. +iterations: 3 + +# Gate before any steps run: per robot, MAVROS connected, then +# local_position/odom publishing (PX4 EKF converged = ready to arm). +ready: + timeout_s: 600 + +# One mcap per robot per iteration; open the .mcap directly in Foxglove. +# {robot} expands to robot_1, robot_2, ... per robot. +record: + enabled: true + topics: + - /tf + - /tf_static + - /{robot}/odometry + - /{robot}/interface/mavros/local_position/odom + - /{robot}/odom_ground_truth + # Or record everything (large — includes camera/LiDAR if running): + # all: true + +# continue | abort_iteration (default) | abort_mission +on_step_failure: abort_iteration + +steps: + - action: + task: takeoff # → /robot_N/tasks/takeoff (TakeoffTask) + goal: {target_altitude_m: 10.0, velocity_m_s: 1.0} + timeout_s: 120 + robots: all # or e.g. [1, 3] + + - wait: 30 # hover + + - run: # arbitrary command escape hatch + container: robot_1 # robot_N = exec on robot N's DDS domain + cmd: ros2 topic echo --once /robot_1/odometry + timeout_s: 20 + + - action: + task: land # → /robot_N/tasks/land (LandTask) + goal: {velocity_m_s: 0.5} + timeout_s: 120 diff --git a/osmo/workflows/airstack-mission.yaml b/osmo/workflows/airstack-mission.yaml new file mode 100644 index 000000000..135972352 --- /dev/null +++ b/osmo/workflows/airstack-mission.yaml @@ -0,0 +1,92 @@ +# AirStack batch mission workflow on OSMO. +# +# Same workspace image and pod layout as airstack-dev.yaml, but instead of a +# single `airstack up` for an interactive session, the entrypoint hands off +# to osmo/workspace/mission_runner.py, which executes the mission spec named +# by OSMO_MISSION_FILE: repeated cycles of +# +# airstack down → airstack up → wait for PX4 ready → record mcap bags +# → run steps (takeoff / land / navigate / any ros2 command) → collect +# bags + logs → airstack down +# +# Mission specs live in osmo/missions/ (cloned with the branch, so what you +# push is what runs). Schema: osmo/missions/README.md. +# +# To submit (or use the wrapper: airstack osmo:mission ): +# +# osmo workflow submit osmo/workflows/airstack-mission.yaml \ +# --pool \ +# --set-env "SSH_PUB_KEY=$(cat ~/.ssh/id_ed25519.pub)" \ +# "OSMO_MISSION_FILE=osmo/missions/example_takeoff_land.yaml" \ +# "AIRSTACK_BRANCH=" +# +# Results land in /osmo/output/airstack-mission-results (symlinked from +# /root/AirStack/osmo/results). With the default OSMO_MISSION_KEEP_ALIVE=true +# the pod stays alive after the mission so you can pull artifacts from your +# laptop: +# +# airstack osmo:fetch ./results/ +# +# Set OSMO_MISSION_KEEP_ALIVE=false for fire-and-forget runs: the task exits +# cleanly when the mission ends, which frees the GPU and triggers the +# `outputs:` upload below (if a destination bucket is configured). + +workflow: + name: airstack-mission + groups: + - name: airstack + tasks: + - name: workspace + lead: true + image: airlab-docker.andrew.cmu.edu/airstack/airstack-osmo-workspace:latest + # Required so the inner dockerd can run (same DinD setup as airstack-dev). + privileged: true + command: ["bash"] + args: ["/usr/local/bin/entrypoint.sh"] + environment: + # Mission selection — repo-relative path into the clone. Override at + # submit time to run a different mission. + OSMO_MISSION_FILE: "osmo/missions/example_takeoff_land.yaml" + # true → pod sleeps after the mission; fetch results over ssh, then + # `airstack osmo:down` (results die with the pod!). + # false → task exits when the mission ends: GPU freed, /osmo/output + # uploaded to `outputs:` destinations. + OSMO_MISSION_KEEP_ALIVE: "true" + AIRSTACK_BRANCH: "main" # branch entrypoint.sh clones + AIRSTACK_REPO_URL: "https://github.com/castacks/AirStack.git" + # SSH_PUB_KEY is supplied at submit time (needed for osmo:fetch): + # --set-env "SSH_PUB_KEY=$(cat ~/.ssh/id_ed25519.pub)" + # Uncomment and point at a bucket the OSMO deployment can write to, to + # have /osmo/output uploaded automatically when the task exits (only + # meaningful with OSMO_MISSION_KEEP_ALIVE=false — a canceled workflow + # does not upload outputs): + # + # outputs: + # - url: s3:///airstack-missions/{{workflow_id}}/ + credentials: + # Same per-user credentials as airstack-dev.yaml — see + # docs/tutorials/airstack_on_osmo.md "Step 0" / `airstack osmo:setup`. + airlab-nucleus: + OMNI_USER: omni_user + OMNI_PASS: omni_pass + OMNI_SERVER: omni_server + airlab-docker-login: + AIRLAB_REGISTRY_USER: username + AIRLAB_REGISTRY_PASS: password + + resources: + default: + cpu: 16 + gpu: 1 + memory: 64Gi + # Same sizing rationale as airstack-dev.yaml (inner image set alone + # exceeds 100Gi extracted), plus headroom for per-iteration mcap + # recordings — bump if a mission records camera/LiDAR over many + # iterations. + storage: 500Gi + + timeout: + # Missions run unattended and multi-iteration; 24h leaves room for long + # batches plus result download time in keep-alive mode. The pod is gone + # when this expires — fetch results first. + exec_timeout: 24h diff --git a/osmo/workspace/Dockerfile b/osmo/workspace/Dockerfile index e80f3be59..0e10d9e49 100644 --- a/osmo/workspace/Dockerfile +++ b/osmo/workspace/Dockerfile @@ -42,6 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ procps \ python3 \ python3-pip \ + python3-yaml \ rsync \ sudo \ tmux \ diff --git a/osmo/workspace/entrypoint.sh b/osmo/workspace/entrypoint.sh index bef9d9ba8..20d89910b 100755 --- a/osmo/workspace/entrypoint.sh +++ b/osmo/workspace/entrypoint.sh @@ -241,36 +241,70 @@ else log "WARN: --payload username= password=" fi -# ─── 6. airstack up ──────────────────────────────────────────────────────── - -# Honor optional overrides passed in via OSMO env. Defaults match a "single -# robot, Isaac Sim with WebRTC livestream" dev session. -export AUTOLAUNCH="${AUTOLAUNCH:-true}" -export NUM_ROBOTS="${NUM_ROBOTS:-1}" -export ISAAC_SIM_LIVESTREAM="${ISAAC_SIM_LIVESTREAM:-true}" - -# COMPOSE_PROFILES selection: the default `desktop,isaac-sim` from .env runs -# the standard isaac-sim service. If the student wants livestream, they (or -# we) swap to the isaac-sim-livestream profile, which is the OSMO-friendly -# variant defined in simulation/isaac-sim/docker/docker-compose.yaml. -if [ "$ISAAC_SIM_LIVESTREAM" = "true" ]; then - export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim-livestream}" -else - export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim}" -fi +# ─── 6. airstack up / mission mode ───────────────────────────────────────── cd "$AIRSTACK_ROOT" -if [ "${OSMO_AIRSTACK_UP:-true}" = "true" ]; then - log "airstack up (COMPOSE_PROFILES=$COMPOSE_PROFILES, NUM_ROBOTS=$NUM_ROBOTS, livestream=$ISAAC_SIM_LIVESTREAM)" - ./airstack.sh up || log "WARN: airstack up exited non-zero — pod stays alive for debugging via SSH" + +if [ -n "${OSMO_MISSION_FILE:-}" ]; then + # Mission mode (airstack-mission.yaml): mission_runner.py owns the full + # stack lifecycle — repeated `airstack down/up` per iteration, PX4 + # readiness gating, mcap recording, and artifact collection into + # /osmo/output. Stack env (NUM_ROBOTS, COMPOSE_PROFILES, + # ISAAC_SIM_SCRIPT_NAME, ...) comes from the mission spec's `env:` block, + # so none of the dev-session defaults below are exported here. + # + # The runner is taken from the clone, not baked into the image, so the + # mission spec and the code executing it always come from the same branch. + MISSION_PATH="$AIRSTACK_ROOT/$OSMO_MISSION_FILE" + [ -f "$MISSION_PATH" ] || MISSION_PATH="$OSMO_MISSION_FILE" # allow absolute paths + if [ ! -f "$MISSION_PATH" ]; then + log "ERROR: mission file not found: $OSMO_MISSION_FILE — pod stays alive for debugging via SSH" + else + log "mission mode: python3 osmo/workspace/mission_runner.py $MISSION_PATH" + if python3 "$AIRSTACK_ROOT/osmo/workspace/mission_runner.py" "$MISSION_PATH" \ + --airstack-root "$AIRSTACK_ROOT"; then + log "mission runner finished: all iterations passed" + else + log "WARN: mission runner exited non-zero — see summary.json / steps.json in the results dir" + fi + fi + if [ "${OSMO_MISSION_KEEP_ALIVE:-true}" != "true" ]; then + log "OSMO_MISSION_KEEP_ALIVE=false — exiting so OSMO uploads /osmo/output and frees the GPU" + # Exit 0 regardless of mission status: a clean task exit is what + # triggers the `outputs:` upload; mission pass/fail lives in summary.json. + exit 0 + fi + log "OSMO_MISSION_KEEP_ALIVE=true — pod stays alive; download results with 'airstack osmo:fetch'" else - log "OSMO_AIRSTACK_UP=false — skipping airstack up; SSH in and run ./airstack.sh up manually" + # Dev mode (airstack-dev.yaml): single bring-up, then the student drives. + # Honor optional overrides passed in via OSMO env. Defaults match a + # "single robot, Isaac Sim with WebRTC livestream" dev session. + export AUTOLAUNCH="${AUTOLAUNCH:-true}" + export NUM_ROBOTS="${NUM_ROBOTS:-1}" + export ISAAC_SIM_LIVESTREAM="${ISAAC_SIM_LIVESTREAM:-true}" + + # COMPOSE_PROFILES selection: the default `desktop,isaac-sim` from .env runs + # the standard isaac-sim service. If the student wants livestream, they (or + # we) swap to the isaac-sim-livestream profile, which is the OSMO-friendly + # variant defined in simulation/isaac-sim/docker/docker-compose.yaml. + if [ "$ISAAC_SIM_LIVESTREAM" = "true" ]; then + export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim-livestream}" + else + export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim}" + fi + + if [ "${OSMO_AIRSTACK_UP:-true}" = "true" ]; then + log "airstack up (COMPOSE_PROFILES=$COMPOSE_PROFILES, NUM_ROBOTS=$NUM_ROBOTS, livestream=$ISAAC_SIM_LIVESTREAM)" + ./airstack.sh up || log "WARN: airstack up exited non-zero — pod stays alive for debugging via SSH" + else + log "OSMO_AIRSTACK_UP=false — skipping airstack up; SSH in and run ./airstack.sh up manually" + fi fi # ─── 7. Sleep ────────────────────────────────────────────────────────────── log "entrypoint complete; sleeping forever so port-forwards keep working" -if [ "$ISAAC_SIM_LIVESTREAM" = "true" ]; then +if [ "${ISAAC_SIM_LIVESTREAM:-false}" = "true" ]; then isaac_sim_log_container="isaac-sim-livestream" else isaac_sim_log_container="airstack-isaac-sim-1" diff --git a/osmo/workspace/mission_runner.py b/osmo/workspace/mission_runner.py new file mode 100644 index 000000000..8d6f873b9 --- /dev/null +++ b/osmo/workspace/mission_runner.py @@ -0,0 +1,684 @@ +#!/usr/bin/env python3 +"""mission_runner.py — batch mission executor for AirStack-on-OSMO pods. + +Reads a declarative mission spec (YAML, see osmo/missions/README.md) and runs +N full iterations of: + + airstack down → airstack up → wait for PX4 ready → start mcap recording + → execute steps (ros2 action goals / topic pubs / service calls / raw + commands / waits) → stop recording → collect bags + container logs → + airstack down + +Artifacts land under one results root per mission run: + + /// + ├── summary.json # per-iteration pass/fail + durations + └── iter_001/ + ├── bags/robot_1/*.mcap # Foxglove-ready (open the .mcap directly) + ├── logs/.log # docker logs snapshot, per container + ├── ready.json # per-robot PX4 readiness timings + └── steps.json # per-step command, output tail, status + +The results root prefers /osmo/output (OSMO uploads that directory to the +workflow's `outputs:` destinations when the task exits) and falls back to +/osmo/results. Either way a symlink is left at +/osmo/results so `airstack osmo:fetch` always finds it. + +Designed to run on the OSMO workspace pod (invoked by entrypoint.sh when +OSMO_MISSION_FILE is set), but has no OSMO dependency: it only needs docker, +python3 + PyYAML, and a checkout with airstack.sh — so it can be tested on +any dev machine that can run `airstack up`. + +Command patterns (ros2 exec into the robot container with a per-robot +ROS_DOMAIN_ID, two-gate PX4 readiness, action-result parsing) mirror +tests/conftest.py and tests/system/test_takeoff_hover_land.py. +""" + +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path + +import yaml + +ROS_DISTRO_SETUP = "/opt/ros/jazzy/setup.bash" +DEFAULT_ROBOT_SETUP_BASH = "/root/AirStack/robot/ros_ws/install/setup.bash" +ROBOT_CONTAINER_PATTERN = "robot.*desktop" + +# Topics recorded when the mission spec doesn't list its own. {robot} +# expands to robot_ per robot. /tf + /tf_static are what let a Foxglove +# 3D panel pose anything at all during replay. +DEFAULT_RECORD_TOPICS = [ + "/tf", + "/tf_static", + "/{robot}/odometry", + "/{robot}/interface/mavros/local_position/odom", + "/{robot}/odom_ground_truth", + "/{robot}/global_plan", +] + +# In-container staging dir for bag recordings (docker cp'd out before the +# stack goes down). Lives in robot container 1 regardless of robot count — +# all replicas share the bridge network, so any container reaches any +# robot's DDS domain by exporting that robot's ROS_DOMAIN_ID. +BAG_STAGING_DIR = "/tmp/osmo_bags" + +# Tasks the GCS action_relay bridges (gcs/ros_ws/src/action_relay). Goals for +# these can be routed `via: gcs` — published as String JSON on +# //tasks//goal (GCS domain 0), exactly the path Foxglove uses. +GCS_RELAY_TASKS = {"takeoff", "land", "navigate", "fixed_trajectory", + "semantic_search", "exploration"} + +MISSION_DEFAULTS = { + "iterations": 1, + "on_step_failure": "abort_iteration", # continue | abort_iteration | abort_mission + "ready": {"timeout_s": 600, "poll_interval_s": 5}, + "record": {"enabled": True}, + # How the stack is brought up. Either (or both): + # services: [isaac-sim, robot-desktop, gcs] → ./airstack.sh up + # (compose auto-enables a named service's profile) + # profiles: [desktop, isaac-sim] → exported as COMPOSE_PROFILES + # With neither, plain `airstack up` with COMPOSE_PROFILES=desktop,isaac-sim. + "stack": {"services": [], "profiles": []}, + # Default route for `action` steps: "gcs" sends goals through the GCS + # action_relay (same path as Foxglove / the GCS panels — exercises the + # full GCS→robot chain incl. the relay's airborne preconditions); + # "robot" sends ros2 action goals directly on the robot's DDS domain. + # Override per step with `via:`. + "command_route": "gcs", + "up_timeout_s": 3600, # first `up` on a fresh pod pulls the full image set + "down_timeout_s": 300, + "robot_setup_bash": DEFAULT_ROBOT_SETUP_BASH, +} + + +def log(msg): + print(f"[mission {datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True) + + +def utc_stamp(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S") + + +# ── subprocess / docker plumbing ─────────────────────────────────────────── + +def sh(cmd_list, timeout, env=None, cwd=None): + """Run a command, capturing output. Never raises on non-zero exit; a + TimeoutExpired is converted into a synthetic failed result so step + failures stay data, not exceptions.""" + try: + return subprocess.run(cmd_list, capture_output=True, text=True, + timeout=timeout, env=env, cwd=cwd) + except subprocess.TimeoutExpired as e: + return subprocess.CompletedProcess( + cmd_list, returncode=124, + stdout=(e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or ""), + stderr=f"timed out after {timeout}s", + ) + + +def docker_exec(container, cmd, timeout): + return sh(["docker", "exec", container, "bash", "-c", cmd], timeout=timeout) + + +def ros2_env_prefix(setup_bash, domain_id): + # Workspace setup is conditional: the GCS container has a different (or + # no) workspace at the robot path, and plain std_msgs pub/echo only needs + # the distro setup anyway. + return (f"source {ROS_DISTRO_SETUP} && " + f"{{ [ -f {setup_bash} ] && source {setup_bash}; }}; " + f"export ROS_DOMAIN_ID={domain_id}") + + +def ros2_exec(container, ros2_cmd, domain_id, setup_bash, timeout): + return docker_exec(container, f"{ros2_env_prefix(setup_bash, domain_id)} && {ros2_cmd}", + timeout=timeout) + + +def list_containers(name_pattern=None, all_states=False): + cmd = ["docker", "ps", "--format", "{{.Names}}"] + if all_states: + cmd.insert(2, "-a") + if name_pattern: + cmd += ["--filter", f"name={name_pattern}"] + result = sh(cmd, timeout=15) + return [n for n in result.stdout.strip().splitlines() if n] + + +def robot_containers(): + """Running robot containers sorted by replica index (replica n ↔ robot_n).""" + def index(name): + tail = name.rsplit("-", 1)[-1] + return int(tail) if tail.isdigit() else 0 + return sorted(list_containers(ROBOT_CONTAINER_PATTERN), key=index) + + +def gcs_container(): + names = list_containers("gcs") + return names[0] if names else None + + +# ── mission spec ─────────────────────────────────────────────────────────── + +def load_mission(path): + with open(path, encoding="utf-8") as f: + spec = yaml.safe_load(f) + if not isinstance(spec, dict): + raise ValueError(f"{path}: mission spec must be a YAML mapping") + if not spec.get("steps"): + raise ValueError(f"{path}: mission spec has no 'steps'") + merged = dict(MISSION_DEFAULTS) + for key, default in MISSION_DEFAULTS.items(): + if isinstance(default, dict): + merged[key] = {**default, **(spec.get(key) or {})} + merged.update({k: v for k, v in spec.items() if not isinstance(MISSION_DEFAULTS.get(k), dict)}) + merged.setdefault("name", Path(path).stem) + merged.setdefault("env", {}) + if merged["on_step_failure"] not in ("continue", "abort_iteration", "abort_mission"): + raise ValueError(f"on_step_failure must be continue|abort_iteration|abort_mission, " + f"got {merged['on_step_failure']!r}") + if merged["command_route"] not in ("gcs", "robot"): + raise ValueError(f"command_route must be gcs|robot, got {merged['command_route']!r}") + for step in merged["steps"]: + action = step.get("action") if isinstance(step, dict) else None + if action: + via = action.get("via", merged["command_route"]) + if via not in ("gcs", "robot"): + raise ValueError(f"action via must be gcs|robot, got {via!r}") + if via == "gcs" and action.get("task") not in GCS_RELAY_TASKS: + raise ValueError( + f"task '{action.get('task')}' is not bridged by the GCS action_relay " + f"({', '.join(sorted(GCS_RELAY_TASKS))}); add `via: robot` to send it " + f"directly on the robot's domain") + return merged + + +def task_action_type(task): + """tasks/semantic_search → task_msgs/action/SemanticSearchTask""" + camel = "".join(part.capitalize() for part in task.split("_")) + return f"task_msgs/action/{camel}Task" + + +def expand(text, n): + """Substitute per-robot placeholders: {n} → robot index, {robot} → robot_.""" + return text.replace("{robot}", f"robot_{n}").replace("{n}", str(n)) + + +def step_robots(step_spec, num_robots): + robots = step_spec.get("robots", "all") + if robots == "all": + return list(range(1, num_robots + 1)) + return [int(r) for r in robots] + + +# ── stack lifecycle ──────────────────────────────────────────────────────── + +class Stack: + def __init__(self, airstack_root, mission): + self.root = airstack_root + self.mission = mission + self.env = os.environ.copy() + self.env.update({k: str(v) for k, v in mission["env"].items()}) + self.env.setdefault("AUTOLAUNCH", "true") + self.env.setdefault("NUM_ROBOTS", "1") + # Bring-up selection (see MISSION_DEFAULTS["stack"]): an explicit + # service list is passed to `airstack up `; explicit + # profiles become COMPOSE_PROFILES. With neither, default profiles. + self.services = [str(s) for s in mission["stack"].get("services") or []] + profiles = [str(p) for p in mission["stack"].get("profiles") or []] + if profiles: + self.env["COMPOSE_PROFILES"] = ",".join(profiles) + elif not self.services: + self.env.setdefault("COMPOSE_PROFILES", "desktop,isaac-sim") + self.num_robots = int(self.env["NUM_ROBOTS"]) + self.setup_bash = mission["robot_setup_bash"] + + def _airstack(self, verb, timeout, extra_args=()): + log(f"airstack {verb} {' '.join(extra_args)} " + f"(NUM_ROBOTS={self.env['NUM_ROBOTS']}, " + f"COMPOSE_PROFILES={self.env.get('COMPOSE_PROFILES', '')})") + return sh([str(Path(self.root) / "airstack.sh"), verb, *extra_args], + timeout=timeout, env=self.env, cwd=self.root) + + def up(self): + result = self._airstack("up", self.mission["up_timeout_s"], + extra_args=self.services) + if result.returncode != 0: + raise RuntimeError(f"airstack up failed (exit {result.returncode}):\n" + + "\n".join(result.stdout.splitlines()[-20:]) + + "\n" + "\n".join(result.stderr.splitlines()[-20:])) + deadline = time.time() + 120 + while time.time() < deadline: + containers = robot_containers() + if len(containers) >= self.num_robots: + return containers + time.sleep(3) + raise RuntimeError(f"expected {self.num_robots} robot containers, " + f"found {robot_containers()} after airstack up") + + def down(self): + result = self._airstack("down", self.mission["down_timeout_s"]) + if result.returncode != 0: + log(f"WARN: airstack down exited {result.returncode}") + + def wait_ready(self, container): + """Two sequential gates per robot (same as the system tests): + 1. mavros/state reports connected=True (MAVROS ↔ PX4 heartbeat); + 2. local_position/odom publishes (PX4 EKF converged — the actual + precondition for arming; `connected` alone fires ~25s too early). + Returns {robot_n: seconds_to_ready}; raises on timeout.""" + cfg = self.mission["ready"] + started = time.time() + deadline = started + cfg["timeout_s"] + connected, ready_at = set(), {} + pending = list(range(1, self.num_robots + 1)) + + while pending and time.time() < deadline: + for n in list(pending): + if n not in connected: + r = ros2_exec(container, + f"timeout 5 ros2 topic echo --once --csv " + f"--field connected /robot_{n}/interface/mavros/state", + domain_id=n, setup_bash=self.setup_bash, timeout=15) + if any(line.strip() == "True" for line in r.stdout.splitlines()): + connected.add(n) + else: + continue + r = ros2_exec(container, + f"timeout 5 ros2 topic echo --once " + f"/robot_{n}/interface/mavros/local_position/odom", + domain_id=n, setup_bash=self.setup_bash, timeout=15) + if r.returncode == 0 and "pose:" in r.stdout: + ready_at[n] = round(time.time() - started, 2) + pending.remove(n) + if pending: + log(f"waiting for PX4: connected={sorted(connected)} pending={pending} " + f"elapsed={time.time() - started:.0f}s") + time.sleep(cfg["poll_interval_s"]) + + if pending: + raise RuntimeError(f"robots {pending} not ready within {cfg['timeout_s']}s " + f"(connected so far: {sorted(connected)})") + log(f"PX4 ready: {ready_at}") + return ready_at + + +# ── bag recording ────────────────────────────────────────────────────────── + +class Recorder: + """One `ros2 bag record -s mcap` per robot, all inside robot container 1. + + Each recorder is started detached with its PID dropped to a file, and + stopped with SIGTERM so rosbag2 finalizes the mcap cleanly. (Not SIGINT: + jobs backgrounded from a non-interactive shell have SIGINT set to + SIG_IGN, so it would never be delivered; rosbag2 handles SIGTERM the + same way.) Bags are docker cp'd to the host before the stack goes down.""" + + def __init__(self, container, mission, num_robots, setup_bash): + self.container = container + self.cfg = mission["record"] + self.num_robots = num_robots + self.setup_bash = setup_bash + self.active = [] + + def start(self): + if not self.cfg.get("enabled", True): + log("recording disabled by mission spec") + return + docker_exec(self.container, + f"rm -rf {BAG_STAGING_DIR} && mkdir -p {BAG_STAGING_DIR}", timeout=15) + for n in range(1, self.num_robots + 1): + if self.cfg.get("all"): + selection = "-a" + else: + topics = [expand(t, n) for t in self.cfg.get("topics", DEFAULT_RECORD_TOPICS)] + selection = " ".join(shlex.quote(t) for t in topics) + out_dir = f"{BAG_STAGING_DIR}/robot_{n}" + inner = ( + f"{ros2_env_prefix(self.setup_bash, n)} && " + # nohup + pidfile: the record process must outlive this + # docker exec; --include-hidden-topics is not needed, and + # unknown listed topics are fine (record waits for them). + f"nohup ros2 bag record -s mcap -o {out_dir} {selection} " + f"> {BAG_STAGING_DIR}/record_{n}.log 2>&1 & " + f"echo $! > {BAG_STAGING_DIR}/record_{n}.pid" + ) + r = docker_exec(self.container, inner, timeout=30) + if r.returncode == 0: + self.active.append(n) + log(f"recording robot_{n} → {out_dir} " + f"({'all topics' if self.cfg.get('all') else f'{len(selection.split())} topics'})") + else: + log(f"WARN: failed to start recorder for robot_{n}: {r.stderr.strip()[:200]}") + + def stop(self): + for n in self.active: + docker_exec(self.container, f""" + pid=$(cat {BAG_STAGING_DIR}/record_{n}.pid 2>/dev/null) || exit 0 + kill -TERM "$pid" 2>/dev/null || exit 0 + for i in $(seq 1 20); do + kill -0 "$pid" 2>/dev/null || exit 0 + sleep 1 + done + kill -9 "$pid" 2>/dev/null || true + """, timeout=40) + if self.active: + log(f"recorders stopped for robots {self.active}") + self.active = [] + + def collect(self, dest_dir): + if not self.cfg.get("enabled", True): + return + dest_dir.mkdir(parents=True, exist_ok=True) + r = sh(["docker", "cp", f"{self.container}:{BAG_STAGING_DIR}/.", str(dest_dir)], + timeout=1800) + if r.returncode != 0: + log(f"WARN: docker cp of bags failed: {r.stderr.strip()[:200]}") + else: + mcaps = list(dest_dir.rglob("*.mcap")) + log(f"collected {len(mcaps)} mcap file(s) → {dest_dir}") + + +# ── step execution ───────────────────────────────────────────────────────── + +def action_ok(stdout): + """ros2 action send_goal --feedback prints the result as YAML; + AirStack task results carry `success: true` on completion.""" + return "success: true" in stdout + + +def tail(text, lines=15): + return "\n".join((text or "").strip().splitlines()[-lines:]) + + +def run_step(stack, container, step_spec, step_index): + """Execute one step; returns a result dict with ok: bool.""" + record = {"index": step_index, "spec": step_spec, + "started_at": datetime.now(timezone.utc).isoformat()} + t0 = time.time() + + if "wait" in step_spec: + seconds = float(step_spec["wait"]) + log(f"step {step_index}: wait {seconds}s") + time.sleep(seconds) + record.update(type="wait", ok=True) + + elif "action" in step_spec: + spec = step_spec["action"] + task = spec["task"] + via = spec.get("via", stack.mission["command_route"]) + goal = spec.get("goal", {}) + # Normalize the goal to JSON: dicts directly; strings may be ros2-style + # YAML ("{target_altitude_m: 10}") — YAML-parse then re-dump. JSON is + # what the GCS relay requires, and it's a YAML subset so the direct + # send_goal path accepts it too. + goal_obj = yaml.safe_load(goal) if isinstance(goal, str) else goal + goal_json = json.dumps(goal_obj or {}) + timeout = float(spec.get("timeout_s", 120)) + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: action {task} {goal_json} via {via} → robots {robots}") + + if via == "gcs": + gcs = gcs_container() + if not gcs: + record.update(type="action", task=task, via=via, ok=False, + error="no gcs container running — `via: gcs` needs the " + "gcs service up (or use `via: robot`)") + record["duration_s"] = round(time.time() - t0, 2) + log(f"step {step_index}: FAILED (no gcs container)") + return record + + def send(n): + # Same path as Foxglove: publish String JSON on + # //tasks//goal (GCS domain 0); the per-robot + # action_relay forwards it as a typed action goal on domain N + # and reports {"success": ..., "message": ...} on + # .../relay_result. Subscribe to the result *before* + # publishing the goal so a fast result can't be missed. + base = f"/robot_{n}/tasks/{task}" + result_file = f"/tmp/relay_result_{task}_{n}.out" + msg_yaml = json.dumps({"data": expand(goal_json, n)}) + script = ( + f"rm -f {result_file}\n" + f"( timeout {int(timeout)} ros2 topic echo --once --field data " + f"{base}/relay_result > {result_file} 2>&1 ) &\n" + f"sub=$!\n" + f"sleep 3\n" + f"ros2 topic pub --once {base}/goal std_msgs/msg/String " + f"{shlex.quote(msg_yaml)} > /dev/null\n" + f"wait $sub\n" + f"cat {result_file}" + ) + r = ros2_exec(gcs, script, domain_id=0, setup_bash=GCS_SETUP_BASH, + timeout=int(timeout + 30)) + ok = '"success": true' in r.stdout + return n, {"exit": r.returncode, "ok": ok, + "output_tail": tail(r.stdout + r.stderr)} + else: + action_type = spec.get("type", task_action_type(task)) + + def send(n): + cmd = (f"ros2 action send_goal --feedback /robot_{n}/tasks/{task} " + f"{action_type} {shlex.quote(expand(goal_json, n))}") + r = ros2_exec(container, cmd, domain_id=n, setup_bash=stack.setup_bash, + timeout=int(timeout + 15)) + return n, {"exit": r.returncode, "ok": action_ok(r.stdout), + "output_tail": tail(r.stdout + r.stderr)} + + with ThreadPoolExecutor(max_workers=len(robots)) as pool: + results = dict(pool.map(send, robots)) + record.update(type="action", task=task, via=via, per_robot=results, + ok=all(v["ok"] for v in results.values())) + + elif "run" in step_spec: + spec = step_spec["run"] + cmd = spec["cmd"] + timeout = float(spec.get("timeout_s", 60)) + target = spec.get("container", "robot_1") + log(f"step {step_index}: run [{target}] {cmd}") + if target == "pod": + r = sh(["bash", "-c", cmd], timeout=timeout, cwd=stack.root) + else: + # robot_N targets exec into robot container 1 on robot N's DDS + # domain (the containers share one bridge network); any other + # value is taken as a literal container name on domain 0. + if target.startswith("robot_") and target[6:].isdigit(): + n = int(target[6:]) + r = ros2_exec(container, expand(cmd, n), domain_id=n, + setup_bash=stack.setup_bash, timeout=timeout + 15) + else: + r = docker_exec(target, cmd, timeout=timeout + 15) + ok = r.returncode == 0 if spec.get("expect_success", True) else True + record.update(type="run", exit=r.returncode, ok=ok, + output_tail=tail(r.stdout + r.stderr)) + + elif "topic_pub" in step_spec: + spec = step_spec["topic_pub"] + msg = spec.get("msg", {}) + msg_str = msg if isinstance(msg, str) else json.dumps(msg) + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: topic_pub {spec['topic']} → robots {robots}") + results = {} + for n in robots: + cmd = (f"ros2 topic pub --once {expand(spec['topic'], n)} " + f"{spec['type']} {shlex.quote(expand(msg_str, n))}") + r = ros2_exec(container, cmd, domain_id=n, setup_bash=stack.setup_bash, + timeout=float(spec.get("timeout_s", 30)) + 15) + results[n] = {"exit": r.returncode, "ok": r.returncode == 0, + "output_tail": tail(r.stdout + r.stderr)} + record.update(type="topic_pub", per_robot=results, + ok=all(v["ok"] for v in results.values())) + + elif "service_call" in step_spec: + spec = step_spec["service_call"] + req = spec.get("request", {}) + req_str = req if isinstance(req, str) else json.dumps(req) + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: service_call {spec['service']} → robots {robots}") + results = {} + for n in robots: + cmd = (f"ros2 service call {expand(spec['service'], n)} " + f"{spec['type']} {shlex.quote(expand(req_str, n))}") + r = ros2_exec(container, cmd, domain_id=n, setup_bash=stack.setup_bash, + timeout=float(spec.get("timeout_s", 30)) + 15) + results[n] = {"exit": r.returncode, "ok": r.returncode == 0, + "output_tail": tail(r.stdout + r.stderr)} + record.update(type="service_call", per_robot=results, + ok=all(v["ok"] for v in results.values())) + + else: + record.update(type="unknown", ok=False, + error=f"unrecognized step keys: {sorted(step_spec)}") + + record["duration_s"] = round(time.time() - t0, 2) + log(f"step {step_index}: {'OK' if record['ok'] else 'FAILED'} " + f"({record['duration_s']}s)") + return record + + +# ── artifacts ────────────────────────────────────────────────────────────── + +def snapshot_container_logs(dest_dir): + dest_dir.mkdir(parents=True, exist_ok=True) + for name in list_containers(name_pattern="airstack", all_states=True): + r = sh(["docker", "logs", name], timeout=120) + (dest_dir / f"{name}.log").write_text( + (r.stdout or "") + (("\n--- stderr ---\n" + r.stderr) if r.stderr else ""), + encoding="utf-8") + + +def write_json(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, default=str) + "\n", encoding="utf-8") + + +def resolve_results_root(airstack_root): + """Prefer /osmo/output (auto-uploaded by OSMO `outputs:` when the task + exits); keep /osmo/results working in both cases so + `airstack osmo:fetch` has one stable path to rsync.""" + fallback = Path(airstack_root) / "osmo" / "results" + override = os.environ.get("OSMO_RESULTS_ROOT") + if override: + root = Path(override) + root.mkdir(parents=True, exist_ok=True) + return root + osmo_output = Path(os.environ.get("OSMO_OUTPUT_DIR", "/osmo/output")) + if osmo_output.is_dir() and os.access(osmo_output, os.W_OK): + root = osmo_output / "airstack-mission-results" + root.mkdir(parents=True, exist_ok=True) + if not fallback.exists(): + fallback.parent.mkdir(parents=True, exist_ok=True) + fallback.symlink_to(root) + return root + fallback.mkdir(parents=True, exist_ok=True) + return fallback + + +# ── main loop ────────────────────────────────────────────────────────────── + +def run_iteration(stack, mission, iter_dir): + """One full up → ready → record → steps → collect → down cycle. + Returns the iteration summary dict; never raises (failures are data).""" + summary = {"status": "passed", "steps_ok": 0, "steps_failed": 0} + recorder = None + container = None + t0 = time.time() + try: + stack.down() + containers = stack.up() + container = containers[0] + summary["up_duration_s"] = round(time.time() - t0, 2) + + ready_at = stack.wait_ready(container) + write_json(iter_dir / "ready.json", ready_at) + + recorder = Recorder(container, mission, stack.num_robots, stack.setup_bash) + recorder.start() + + steps = [] + for i, step_spec in enumerate(mission["steps"], start=1): + record = run_step(stack, container, step_spec, i) + steps.append(record) + if record["ok"]: + summary["steps_ok"] += 1 + continue + summary["steps_failed"] += 1 + summary["status"] = "failed" + policy = mission["on_step_failure"] + if policy == "continue": + continue + if policy == "abort_mission": + summary["abort_mission"] = True + break + write_json(iter_dir / "steps.json", steps) + + except Exception as e: + summary["status"] = "error" + summary["error"] = str(e) + log(f"ERROR: iteration aborted: {e}") + + finally: + # Artifact collection happens even on failure — a failed flight's + # bag is usually the most interesting one. + if recorder is not None: + recorder.stop() + recorder.collect(iter_dir / "bags") + if container is not None or list_containers("airstack", all_states=True): + snapshot_container_logs(iter_dir / "logs") + stack.down() + summary["duration_s"] = round(time.time() - t0, 2) + write_json(iter_dir / "iteration.json", summary) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("mission_file", help="Path to mission spec YAML") + parser.add_argument("--airstack-root", + default=os.environ.get("AIRSTACK_ROOT", "/root/AirStack")) + parser.add_argument("--dry-run", action="store_true", + help="Validate the spec and print the plan without touching docker") + args = parser.parse_args() + + mission = load_mission(args.mission_file) + stack = Stack(args.airstack_root, mission) + + log(f"mission '{mission['name']}': {mission['iterations']} iteration(s), " + f"{len(mission['steps'])} step(s), {stack.num_robots} robot(s)") + if args.dry_run: + print(yaml.safe_dump(mission, sort_keys=False)) + return 0 + + results_root = resolve_results_root(args.airstack_root) + run_dir = results_root / mission["name"] / utc_stamp() + run_dir.mkdir(parents=True) + log(f"results → {run_dir}") + + iterations = [] + for i in range(1, mission["iterations"] + 1): + log(f"━━━ iteration {i}/{mission['iterations']} ━━━") + iter_dir = run_dir / f"iter_{i:03d}" + summary = run_iteration(stack, mission, iter_dir) + summary["iteration"] = i + iterations.append(summary) + write_json(run_dir / "summary.json", + {"mission": mission["name"], "mission_file": args.mission_file, + "iterations": iterations}) + if summary.get("abort_mission"): + log("on_step_failure=abort_mission — stopping remaining iterations") + break + + passed = sum(1 for s in iterations if s["status"] == "passed") + log(f"mission complete: {passed}/{len(iterations)} iteration(s) passed; " + f"results in {run_dir}") + return 0 if passed == len(iterations) else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 6ad33f0012250a54d83199e5c3f9375a85e53158 Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Wed, 10 Jun 2026 15:54:20 -0400 Subject: [PATCH 02/27] updated example mission --- osmo/missions/example_takeoff_land.yaml | 23 +++++++++++++++++++++-- osmo/workspace/mission_runner.py | 10 ++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml index 537e95b37..1365d44ac 100644 --- a/osmo/missions/example_takeoff_land.yaml +++ b/osmo/missions/example_takeoff_land.yaml @@ -11,11 +11,20 @@ name: example_takeoff_land +# How to bring the stack up. `services` are passed straight through: +# ./airstack.sh up isaac-sim robot-desktop gcs +# (docker compose auto-enables a named service's profile, so no +# COMPOSE_PROFILES needed). On OSMO swap isaac-sim → isaac-sim-livestream to +# watch the run over WebRTC. Alternatively use `profiles:` to export +# COMPOSE_PROFILES and run a plain `airstack up`. +stack: + services: [isaac-sim, robot-desktop, gcs] + # profiles: [desktop, isaac-sim] + # Exported before every `airstack up`. Anything the compose stack reads from # the environment can go here (.env values, ISAAC_SIM_SCRIPT_NAME, etc.). env: NUM_ROBOTS: 1 - COMPOSE_PROFILES: desktop,isaac-sim ISAAC_SIM_HEADLESS: "true" ISAAC_SIM_USE_STANDALONE: "true" # The default script only spawns one drone; for NUM_ROBOTS > 1 switch to @@ -24,6 +33,14 @@ env: # Missions are unattended — the sim must start playing without a GUI click. PLAY_SIM_ON_START: "true" +# Where `action` steps are sent by default: +# gcs — through the GCS action_relay, the same String-JSON → typed-goal +# path Foxglove and the GCS panels use (exercises the full +# GCS→robot chain, including the relay's airborne preconditions) +# robot — `ros2 action send_goal` directly on the robot's DDS domain +# Override per step with `via:`. +command_route: gcs + # Full up → fly → collect → down cycles. iterations: 3 @@ -58,7 +75,8 @@ steps: - wait: 30 # hover - run: # arbitrary command escape hatch - container: robot_1 # robot_N = exec on robot N's DDS domain + container: robot_1 # robot_N = exec on robot N's DDS domain; + # also: gcs | pod | cmd: ros2 topic echo --once /robot_1/odometry timeout_s: 20 @@ -66,3 +84,4 @@ steps: task: land # → /robot_N/tasks/land (LandTask) goal: {velocity_m_s: 0.5} timeout_s: 120 + # via: robot # bypass the GCS relay for this step diff --git a/osmo/workspace/mission_runner.py b/osmo/workspace/mission_runner.py index 8d6f873b9..5e36dd544 100644 --- a/osmo/workspace/mission_runner.py +++ b/osmo/workspace/mission_runner.py @@ -49,6 +49,7 @@ ROS_DISTRO_SETUP = "/opt/ros/jazzy/setup.bash" DEFAULT_ROBOT_SETUP_BASH = "/root/AirStack/robot/ros_ws/install/setup.bash" +GCS_SETUP_BASH = "/root/AirStack/gcs/ros_ws/install/setup.bash" ROBOT_CONTAINER_PATTERN = "robot.*desktop" # Topics recorded when the mission spec doesn't list its own. {robot} @@ -484,6 +485,15 @@ def send(n): log(f"step {step_index}: run [{target}] {cmd}") if target == "pod": r = sh(["bash", "-c", cmd], timeout=timeout, cwd=stack.root) + elif target == "gcs": + g = gcs_container() + if g: + r = ros2_exec(g, cmd, domain_id=0, setup_bash=GCS_SETUP_BASH, + timeout=timeout + 15) + else: + r = subprocess.CompletedProcess( + cmd, returncode=1, stdout="", + stderr="no gcs container running") else: # robot_N targets exec into robot container 1 on robot N's DDS # domain (the containers share one bridge network); any other From 55be3f80f889dfb37ad1649434df01eeb6822482 Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Wed, 10 Jun 2026 16:05:46 -0400 Subject: [PATCH 03/27] changes to recording topics --- osmo/missions/example_takeoff_land.yaml | 21 ++- osmo/workflows/airstack-mission.yaml | 2 +- osmo/workspace/mission_runner.py | 197 +++++++++++++++++------- 3 files changed, 161 insertions(+), 59 deletions(-) diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml index 1365d44ac..5f3275204 100644 --- a/osmo/missions/example_takeoff_land.yaml +++ b/osmo/missions/example_takeoff_land.yaml @@ -49,17 +49,26 @@ iterations: 3 ready: timeout_s: 600 -# One mcap per robot per iteration; open the .mcap directly in Foxglove. -# {robot} expands to robot_1, robot_2, ... per robot. +# One mcap per iteration with ALL robots' data, recorded on the GCS +# (domain 0) where each robot's domain_bridge forwards its state topics — +# replaying the .mcap in Foxglove reproduces the live GCS view. +# {robot} expands to robot_1..robot_N and the list is unioned. record: enabled: true + scope: gcs # or `robot`: one mcap per robot on its own + # domain, for topics not bridged to the GCS topics: - /tf - /tf_static - - /{robot}/odometry - - /{robot}/interface/mavros/local_position/odom - - /{robot}/odom_ground_truth - # Or record everything (large — includes camera/LiDAR if running): + - /{robot}/odometry_conversion/odometry + - /{robot}/interface/mavros/global_position/global + - /{robot}/trajectory_controller/trajectory_vis + - /{robot}/global_plan + - /gcs/robot_markers + - /gcs/map_origin/location + - /gcs/map_origin/ground_msl + - /gcs/{robot}/location + # Or record everything visible on the recording domain (large): # all: true # continue | abort_iteration (default) | abort_mission diff --git a/osmo/workflows/airstack-mission.yaml b/osmo/workflows/airstack-mission.yaml index 135972352..094ed25aa 100644 --- a/osmo/workflows/airstack-mission.yaml +++ b/osmo/workflows/airstack-mission.yaml @@ -52,7 +52,7 @@ workflow: # false → task exits when the mission ends: GPU freed, /osmo/output # uploaded to `outputs:` destinations. OSMO_MISSION_KEEP_ALIVE: "true" - AIRSTACK_BRANCH: "main" # branch entrypoint.sh clones + AIRSTACK_BRANCH: "krrishj18/osmo-mission-runner" # branch entrypoint.sh clones AIRSTACK_REPO_URL: "https://github.com/castacks/AirStack.git" # SSH_PUB_KEY is supplied at submit time (needed for osmo:fetch): # --set-env "SSH_PUB_KEY=$(cat ~/.ssh/id_ed25519.pub)" diff --git a/osmo/workspace/mission_runner.py b/osmo/workspace/mission_runner.py index 5e36dd544..9d054b070 100644 --- a/osmo/workspace/mission_runner.py +++ b/osmo/workspace/mission_runner.py @@ -52,22 +52,38 @@ GCS_SETUP_BASH = "/root/AirStack/gcs/ros_ws/install/setup.bash" ROBOT_CONTAINER_PATTERN = "robot.*desktop" -# Topics recorded when the mission spec doesn't list its own. {robot} -# expands to robot_ per robot. /tf + /tf_static are what let a Foxglove -# 3D panel pose anything at all during replay. -DEFAULT_RECORD_TOPICS = [ +# Default topics for `record.scope: gcs` — one recorder in the GCS container +# on domain 0, where each robot's domain_bridge (autonomy_bringup +# onboard_all/config/domain_bridge.yaml) forwards its state topics. All +# robots land in ONE mcap, and replaying it in Foxglove reproduces the live +# GCS view (gcs_visualizer markers are already in the global 'map' frame). +# {robot} expands to robot_1..robot_N and the result is unioned. +DEFAULT_GCS_RECORD_TOPICS = [ "/tf", "/tf_static", - "/{robot}/odometry", - "/{robot}/interface/mavros/local_position/odom", - "/{robot}/odom_ground_truth", + "/{robot}/odometry_conversion/odometry", + "/{robot}/interface/mavros/global_position/global", + "/{robot}/trajectory_controller/trajectory_vis", + "/{robot}/global_plan", + "/gcs/robot_markers", + "/gcs/map_origin/location", + "/gcs/map_origin/ground_msl", + "/gcs/{robot}/location", +] + +# Default topics for `record.scope: robot` — one recorder (and one mcap) per +# robot on its own DDS domain. Use this scope for topics that are NOT +# bridged to the GCS (raw sensors, high-rate local topics). +DEFAULT_ROBOT_RECORD_TOPICS = [ + "/tf", + "/tf_static", + "/{robot}/odometry_conversion/odometry", + "/{robot}/interface/mavros/global_position/global", "/{robot}/global_plan", ] # In-container staging dir for bag recordings (docker cp'd out before the -# stack goes down). Lives in robot container 1 regardless of robot count — -# all replicas share the bridge network, so any container reaches any -# robot's DDS domain by exporting that robot's ROS_DOMAIN_ID. +# stack goes down). BAG_STAGING_DIR = "/tmp/osmo_bags" # Tasks the GCS action_relay bridges (gcs/ros_ws/src/action_relay). Goals for @@ -80,7 +96,10 @@ "iterations": 1, "on_step_failure": "abort_iteration", # continue | abort_iteration | abort_mission "ready": {"timeout_s": 600, "poll_interval_s": 5}, - "record": {"enabled": True}, + # scope "gcs": one recorder on GCS domain 0 → one mcap with every robot's + # bridged topics (default). scope "robot": one recorder + one mcap per + # robot on its own domain (for unbridged/high-rate topics). + "record": {"enabled": True, "scope": "gcs"}, # How the stack is brought up. Either (or both): # services: [isaac-sim, robot-desktop, gcs] → ./airstack.sh up # (compose auto-enables a named service's profile) @@ -186,6 +205,9 @@ def load_mission(path): f"got {merged['on_step_failure']!r}") if merged["command_route"] not in ("gcs", "robot"): raise ValueError(f"command_route must be gcs|robot, got {merged['command_route']!r}") + if merged["record"].get("scope", "gcs") not in ("gcs", "robot"): + raise ValueError(f"record.scope must be gcs|robot, " + f"got {merged['record'].get('scope')!r}") for step in merged["steps"]: action = step.get("action") if isinstance(step, dict) else None if action: @@ -218,6 +240,15 @@ def step_robots(step_spec, num_robots): return [int(r) for r in robots] +def uses_gcs_route(mission): + """True if any action step routes through the GCS action_relay.""" + for step in mission["steps"]: + action = step.get("action") if isinstance(step, dict) else None + if action and action.get("via", mission["command_route"]) == "gcs": + return True + return False + + # ── stack lifecycle ──────────────────────────────────────────────────────── class Stack: @@ -307,13 +338,46 @@ def wait_ready(self, container): raise RuntimeError(f"robots {pending} not ready within {cfg['timeout_s']}s " f"(connected so far: {sorted(connected)})") log(f"PX4 ready: {ready_at}") + + # Gate 3 (only when actions route via the GCS): the per-robot + # action_relay nodes must be up on the GCS before goals are sent — + # the relay's goal subscription is volatile, so a goal published + # before the relay exists is silently lost. + if uses_gcs_route(self.mission): + while time.time() < deadline: + gcs = gcs_container() + if gcs: + r = ros2_exec(gcs, "timeout 10 ros2 node list", + domain_id=0, setup_bash=GCS_SETUP_BASH, timeout=20) + missing = [n for n in range(1, self.num_robots + 1) + if f"action_relay_robot_{n}" not in r.stdout] + if not missing: + ready_at["gcs_relay"] = round(time.time() - started, 2) + log(f"GCS action_relay ready ({ready_at['gcs_relay']}s)") + break + log(f"waiting for GCS action_relay: missing robots {missing}") + else: + log("waiting for gcs container") + time.sleep(cfg["poll_interval_s"]) + if "gcs_relay" not in ready_at: + raise RuntimeError( + f"GCS action_relay not ready within {cfg['timeout_s']}s — " + f"is the gcs service in stack.services/profiles and " + f"AUTOLAUNCH=true? (or set command_route: robot)") return ready_at # ── bag recording ────────────────────────────────────────────────────────── class Recorder: - """One `ros2 bag record -s mcap` per robot, all inside robot container 1. + """`ros2 bag record -s mcap` per the mission's record.scope. + + scope "gcs" (default): a single recorder in the GCS container on domain + 0, where every robot's domain_bridge forwards its state topics — one + mcap holds all robots, and Foxglove replay matches the live GCS view. + + scope "robot": one recorder per robot inside robot container 1, each on + that robot's DDS domain — for topics that aren't bridged to the GCS. Each recorder is started detached with its PID dropped to a file, and stopped with SIGTERM so rosbag2 finalizes the mcap cleanly. (Not SIGINT: @@ -321,47 +385,75 @@ class Recorder: SIG_IGN, so it would never be delivered; rosbag2 handles SIGTERM the same way.) Bags are docker cp'd to the host before the stack goes down.""" - def __init__(self, container, mission, num_robots, setup_bash): - self.container = container + def __init__(self, robot_container, mission, num_robots, setup_bash): + self.robot_container = robot_container self.cfg = mission["record"] + self.scope = self.cfg.get("scope", "gcs") self.num_robots = num_robots self.setup_bash = setup_bash + # (container, domain_id, tag) per active recorder. self.active = [] + def _topic_selection(self, robots): + """Build the `ros2 bag record` topic args; `robots` is the list of + robot indices whose {robot}/{n} placeholders to expand (unioned, + order-preserving, deduplicated).""" + if self.cfg.get("all"): + return "-a" + default = (DEFAULT_GCS_RECORD_TOPICS if self.scope == "gcs" + else DEFAULT_ROBOT_RECORD_TOPICS) + topics = [] + for t in self.cfg.get("topics", default): + if "{robot}" in t or "{n}" in t: + topics.extend(expand(t, n) for n in robots) + else: + topics.append(t) + seen = set() + unique = [t for t in topics if not (t in seen or seen.add(t))] + return " ".join(shlex.quote(t) for t in unique) + + def _start_one(self, container, domain_id, tag, selection, setup_bash): + out_dir = f"{BAG_STAGING_DIR}/{tag}" + inner = ( + f"{ros2_env_prefix(setup_bash, domain_id)} && " + f"mkdir -p {BAG_STAGING_DIR} && rm -rf {out_dir} && " + # nohup + pidfile: the record process must outlive this docker + # exec; topics that don't exist yet are fine (record waits). + f"nohup ros2 bag record -s mcap -o {out_dir} {selection} " + f"> {BAG_STAGING_DIR}/record_{tag}.log 2>&1 & " + f"echo $! > {BAG_STAGING_DIR}/record_{tag}.pid" + ) + r = docker_exec(container, inner, timeout=30) + if r.returncode == 0: + self.active.append((container, tag)) + n_topics = "all topics" if selection == "-a" else f"{len(selection.split())} topics" + log(f"recording [{tag}] in {container} (domain {domain_id}) " + f"→ {out_dir} ({n_topics})") + else: + log(f"WARN: failed to start recorder [{tag}]: {r.stderr.strip()[:200]}") + def start(self): if not self.cfg.get("enabled", True): log("recording disabled by mission spec") return - docker_exec(self.container, - f"rm -rf {BAG_STAGING_DIR} && mkdir -p {BAG_STAGING_DIR}", timeout=15) - for n in range(1, self.num_robots + 1): - if self.cfg.get("all"): - selection = "-a" - else: - topics = [expand(t, n) for t in self.cfg.get("topics", DEFAULT_RECORD_TOPICS)] - selection = " ".join(shlex.quote(t) for t in topics) - out_dir = f"{BAG_STAGING_DIR}/robot_{n}" - inner = ( - f"{ros2_env_prefix(self.setup_bash, n)} && " - # nohup + pidfile: the record process must outlive this - # docker exec; --include-hidden-topics is not needed, and - # unknown listed topics are fine (record waits for them). - f"nohup ros2 bag record -s mcap -o {out_dir} {selection} " - f"> {BAG_STAGING_DIR}/record_{n}.log 2>&1 & " - f"echo $! > {BAG_STAGING_DIR}/record_{n}.pid" - ) - r = docker_exec(self.container, inner, timeout=30) - if r.returncode == 0: - self.active.append(n) - log(f"recording robot_{n} → {out_dir} " - f"({'all topics' if self.cfg.get('all') else f'{len(selection.split())} topics'})") - else: - log(f"WARN: failed to start recorder for robot_{n}: {r.stderr.strip()[:200]}") + robots = list(range(1, self.num_robots + 1)) + if self.scope == "gcs": + gcs = gcs_container() + if not gcs: + log("WARN: record.scope is 'gcs' but no gcs container is running — " + "recording skipped (bring up the gcs service or use scope: robot)") + return + self._start_one(gcs, 0, "gcs", self._topic_selection(robots), + GCS_SETUP_BASH) + else: + for n in robots: + self._start_one(self.robot_container, n, f"robot_{n}", + self._topic_selection([n]), self.setup_bash) def stop(self): - for n in self.active: - docker_exec(self.container, f""" - pid=$(cat {BAG_STAGING_DIR}/record_{n}.pid 2>/dev/null) || exit 0 + for container, tag in self.active: + docker_exec(container, f""" + pid=$(cat {BAG_STAGING_DIR}/record_{tag}.pid 2>/dev/null) || exit 0 kill -TERM "$pid" 2>/dev/null || exit 0 for i in $(seq 1 20); do kill -0 "$pid" 2>/dev/null || exit 0 @@ -370,20 +462,21 @@ def stop(self): kill -9 "$pid" 2>/dev/null || true """, timeout=40) if self.active: - log(f"recorders stopped for robots {self.active}") - self.active = [] + log(f"recorders stopped: {[tag for _, tag in self.active]}") def collect(self, dest_dir): - if not self.cfg.get("enabled", True): + if not self.active: return dest_dir.mkdir(parents=True, exist_ok=True) - r = sh(["docker", "cp", f"{self.container}:{BAG_STAGING_DIR}/.", str(dest_dir)], - timeout=1800) - if r.returncode != 0: - log(f"WARN: docker cp of bags failed: {r.stderr.strip()[:200]}") - else: - mcaps = list(dest_dir.rglob("*.mcap")) - log(f"collected {len(mcaps)} mcap file(s) → {dest_dir}") + for container in {c for c, _ in self.active}: + r = sh(["docker", "cp", f"{container}:{BAG_STAGING_DIR}/.", str(dest_dir)], + timeout=1800) + if r.returncode != 0: + log(f"WARN: docker cp of bags from {container} failed: " + f"{r.stderr.strip()[:200]}") + self.active = [] + mcaps = list(dest_dir.rglob("*.mcap")) + log(f"collected {len(mcaps)} mcap file(s) → {dest_dir}") # ── step execution ───────────────────────────────────────────────────────── From 1fdbf8cdc4212bc8a0a53263f73e33c5ac100934 Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Wed, 10 Jun 2026 16:31:53 -0400 Subject: [PATCH 04/27] updates to example mission --- osmo/missions/example_takeoff_land.yaml | 23 ++++---- osmo/workspace/mission_runner.py | 76 +++++++++++++++++-------- 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml index 5f3275204..727c420a1 100644 --- a/osmo/missions/example_takeoff_land.yaml +++ b/osmo/missions/example_takeoff_land.yaml @@ -74,23 +74,26 @@ record: # continue | abort_iteration (default) | abort_mission on_step_failure: abort_iteration +# Steps run top to bottom. Every step that addresses a robot fans out over a +# robot selection — `robots:` (default `all` = robots 1..NUM_ROBOTS, or a +# list like [1, 3]) — and {robot} → robot_N / {n} → N expand per robot. +# Per-step knobs (all optional, shown here with their defaults): +# action: robots=all via= timeout_s=120 +# via: gcs (through the GCS action_relay) | robot (direct send_goal) +# run: robots=all container=robot_{n} (per-robot) or robot_1 | gcs | pod | +# ; fans out when cmd/container references {robot}/{n} +# wait: (runs once) steps: - action: - task: takeoff # → /robot_N/tasks/takeoff (TakeoffTask) + task: takeoff # → //tasks/takeoff (TakeoffTask) goal: {target_altitude_m: 10.0, velocity_m_s: 1.0} - timeout_s: 120 - robots: all # or e.g. [1, 3] - wait: 30 # hover - - run: # arbitrary command escape hatch - container: robot_1 # robot_N = exec on robot N's DDS domain; - # also: gcs | pod | - cmd: ros2 topic echo --once /robot_1/odometry + - run: # arbitrary command, fanned out per robot + cmd: ros2 topic echo --once /{robot}/odometry_conversion/odometry timeout_s: 20 - action: - task: land # → /robot_N/tasks/land (LandTask) + task: land # → //tasks/land (LandTask) goal: {velocity_m_s: 0.5} - timeout_s: 120 - # via: robot # bypass the GCS relay for this step diff --git a/osmo/workspace/mission_runner.py b/osmo/workspace/mission_runner.py index 9d054b070..74e6aa475 100644 --- a/osmo/workspace/mission_runner.py +++ b/osmo/workspace/mission_runner.py @@ -233,6 +233,11 @@ def expand(text, n): return text.replace("{robot}", f"robot_{n}").replace("{n}", str(n)) +def has_placeholder(*strings): + """True if any string contains a per-robot placeholder ({robot} or {n}).""" + return any(s and ("{robot}" in s or "{n}" in s) for s in strings) + + def step_robots(step_spec, num_robots): robots = step_spec.get("robots", "all") if robots == "all": @@ -491,6 +496,34 @@ def tail(text, lines=15): return "\n".join((text or "").strip().splitlines()[-lines:]) +def _run_one(stack, robot_container, target, cmd, timeout, expect_success): + """Run one resolved `run` command. `target` is already robot-expanded: + pod → on the pod itself (cwd = AirStack root) + gcs → in the gcs container on domain 0 + robot_ → in robot container 1 on robot N's DDS domain + → literal container name (domain 0) + """ + if target == "pod": + r = sh(["bash", "-c", cmd], timeout=timeout, cwd=stack.root) + elif target == "gcs": + g = gcs_container() + if g: + r = ros2_exec(g, cmd, domain_id=0, setup_bash=GCS_SETUP_BASH, + timeout=timeout + 15) + else: + r = subprocess.CompletedProcess(cmd, returncode=1, stdout="", + stderr="no gcs container running") + elif target.startswith("robot_") and target[6:].isdigit(): + n = int(target[6:]) + r = ros2_exec(robot_container, cmd, domain_id=n, + setup_bash=stack.setup_bash, timeout=timeout + 15) + else: + r = docker_exec(target, cmd, timeout=timeout + 15) + ok = (r.returncode == 0) if expect_success else True + return {"target": target, "exit": r.returncode, "ok": ok, + "output_tail": tail(r.stdout + r.stderr)} + + def run_step(stack, container, step_spec, step_index): """Execute one step; returns a result dict with ok: bool.""" record = {"index": step_index, "spec": step_spec, @@ -574,32 +607,25 @@ def send(n): spec = step_spec["run"] cmd = spec["cmd"] timeout = float(spec.get("timeout_s", 60)) - target = spec.get("container", "robot_1") - log(f"step {step_index}: run [{target}] {cmd}") - if target == "pod": - r = sh(["bash", "-c", cmd], timeout=timeout, cwd=stack.root) - elif target == "gcs": - g = gcs_container() - if g: - r = ros2_exec(g, cmd, domain_id=0, setup_bash=GCS_SETUP_BASH, - timeout=timeout + 15) - else: - r = subprocess.CompletedProcess( - cmd, returncode=1, stdout="", - stderr="no gcs container running") + expect = spec.get("expect_success", True) + # `container` may reference {n}/{robot} to fan out over robots. If it's + # omitted, default to robot_{n} when the command is per-robot (has a + # placeholder) and robot_1 otherwise. A step fans out over `robots` + # (default all) iff its command or container references {n}/{robot}. + target = spec.get("container") or ("robot_{n}" if has_placeholder(cmd) else "robot_1") + if has_placeholder(cmd, target): + robots = step_robots(spec, stack.num_robots) + log(f"step {step_index}: run [{target}] {cmd} → robots {robots}") + results = {} + for n in robots: + results[n] = _run_one(stack, container, expand(target, n), + expand(cmd, n), timeout, expect) + record.update(type="run", per_robot=results, + ok=all(v["ok"] for v in results.values())) else: - # robot_N targets exec into robot container 1 on robot N's DDS - # domain (the containers share one bridge network); any other - # value is taken as a literal container name on domain 0. - if target.startswith("robot_") and target[6:].isdigit(): - n = int(target[6:]) - r = ros2_exec(container, expand(cmd, n), domain_id=n, - setup_bash=stack.setup_bash, timeout=timeout + 15) - else: - r = docker_exec(target, cmd, timeout=timeout + 15) - ok = r.returncode == 0 if spec.get("expect_success", True) else True - record.update(type="run", exit=r.returncode, ok=ok, - output_tail=tail(r.stdout + r.stderr)) + log(f"step {step_index}: run [{target}] {cmd}") + res = _run_one(stack, container, target, cmd, timeout, expect) + record.update(type="run", **res) elif "topic_pub" in step_spec: spec = step_spec["topic_pub"] From 61b673be32ffcd2fcad51346a56b1b6dbdd4da2c Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Thu, 11 Jun 2026 15:30:11 -0400 Subject: [PATCH 05/27] fixed mission runner script --- osmo/missions/example_takeoff_land.yaml | 15 +++- osmo/workflows/airstack-mission.yaml | 36 ++++++++-- osmo/workspace/entrypoint.sh | 83 +++++++-------------- osmo/workspace/mission_launcher.sh | 96 +++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 63 deletions(-) create mode 100755 osmo/workspace/mission_launcher.sh diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml index 727c420a1..1eaa96554 100644 --- a/osmo/missions/example_takeoff_land.yaml +++ b/osmo/missions/example_takeoff_land.yaml @@ -18,7 +18,7 @@ name: example_takeoff_land # watch the run over WebRTC. Alternatively use `profiles:` to export # COMPOSE_PROFILES and run a plain `airstack up`. stack: - services: [isaac-sim, robot-desktop, gcs] + services: [isaac-sim-livestream, robot-desktop, gcs] # profiles: [desktop, isaac-sim] # Exported before every `airstack up`. Anything the compose stack reads from @@ -87,12 +87,23 @@ steps: - action: task: takeoff # → //tasks/takeoff (TakeoffTask) goal: {target_altitude_m: 10.0, velocity_m_s: 1.0} + #to specify robot(s) + #robots: [1] (1st robot) + #robots: [1, 3] (1st and 3rd robot) + #robots: all (all robots) - wait: 30 # hover - - run: # arbitrary command, fanned out per robot + - run: # arbitrary command, that runs per robot cmd: ros2 topic echo --once /{robot}/odometry_conversion/odometry timeout_s: 20 + #to specify robot(s): + #robots: [1] (1st robot) + #robots: [1, 3] (1st and 3rd robot) + + #To specify the container: + #container: gcs (GCS container) + #container: - action: task: land # → //tasks/land (LandTask) diff --git a/osmo/workflows/airstack-mission.yaml b/osmo/workflows/airstack-mission.yaml index 094ed25aa..80e79a659 100644 --- a/osmo/workflows/airstack-mission.yaml +++ b/osmo/workflows/airstack-mission.yaml @@ -1,14 +1,22 @@ # AirStack batch mission workflow on OSMO. # # Same workspace image and pod layout as airstack-dev.yaml, but instead of a -# single `airstack up` for an interactive session, the entrypoint hands off -# to osmo/workspace/mission_runner.py, which executes the mission spec named -# by OSMO_MISSION_FILE: repeated cycles of +# single `airstack up` for an interactive session, this runs a batch mission: +# osmo/workspace/mission_runner.py executes the mission spec named by +# OSMO_MISSION_FILE — repeated cycles of # # airstack down → airstack up → wait for PX4 ready → record mcap bags # → run steps (takeoff / land / navigate / any ros2 command) → collect # bags + logs → airstack down # +# NO IMAGE REBUILD NEEDED. The mission engine is NOT baked into the image: +# the command below reuses the baked entrypoint for pod setup only +# (OSMO_AIRSTACK_UP=false → sshd, inner dockerd, branch clone, creds, registry +# login — no `airstack up`), then hands off to mission_launcher.sh FROM THE +# CLONE. So `git push` your branch and resubmit; the launcher + runner + spec +# all come from the branch. (The launcher pip-installs PyYAML at runtime if +# the image predates it.) +# # Mission specs live in osmo/missions/ (cloned with the branch, so what you # push is what runs). Schema: osmo/missions/README.md. # @@ -41,8 +49,26 @@ workflow: image: airlab-docker.andrew.cmu.edu/airstack/airstack-osmo-workspace:latest # Required so the inner dockerd can run (same DinD setup as airstack-dev). privileged: true - command: ["bash"] - args: ["/usr/local/bin/entrypoint.sh"] + # Bootstrap: run the baked entrypoint for pod setup ONLY + # (OSMO_AIRSTACK_UP=false), backgrounded, then hand off to the launcher + # that ships in the freshly-cloned branch. This is what makes the + # mission engine rebuild-free — see the header comment. + command: ["bash", "-c"] + args: + - | + set -uo pipefail + log() { echo "[mission-bootstrap] $*"; } + log "starting pod setup via baked entrypoint (OSMO_AIRSTACK_UP=false)" + OSMO_AIRSTACK_UP=false /usr/local/bin/entrypoint.sh & + LAUNCHER=/root/AirStack/osmo/workspace/mission_launcher.sh + log "waiting for branch clone to provide $LAUNCHER" + for i in $(seq 1 600); do [ -f "$LAUNCHER" ] && break; sleep 2; done + if [ ! -f "$LAUNCHER" ]; then + log "ERROR: $LAUNCHER not found after clone wait; sleeping for SSH debug" + exec sleep infinity + fi + log "handing off to mission_launcher.sh" + exec bash "$LAUNCHER" environment: # Mission selection — repo-relative path into the clone. Override at # submit time to run a different mission. diff --git a/osmo/workspace/entrypoint.sh b/osmo/workspace/entrypoint.sh index 20d89910b..dd53a94ae 100755 --- a/osmo/workspace/entrypoint.sh +++ b/osmo/workspace/entrypoint.sh @@ -241,70 +241,41 @@ else log "WARN: --payload username= password=" fi -# ─── 6. airstack up / mission mode ───────────────────────────────────────── +# ─── 6. airstack up ──────────────────────────────────────────────────────── + +# Honor optional overrides passed in via OSMO env. Defaults match a "single +# robot, Isaac Sim with WebRTC livestream" dev session. +export AUTOLAUNCH="${AUTOLAUNCH:-true}" +export NUM_ROBOTS="${NUM_ROBOTS:-1}" +export ISAAC_SIM_LIVESTREAM="${ISAAC_SIM_LIVESTREAM:-true}" + +# COMPOSE_PROFILES selection: the default `desktop,isaac-sim` from .env runs +# the standard isaac-sim service. If the student wants livestream, they (or +# we) swap to the isaac-sim-livestream profile, which is the OSMO-friendly +# variant defined in simulation/isaac-sim/docker/docker-compose.yaml. +if [ "$ISAAC_SIM_LIVESTREAM" = "true" ]; then + export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim-livestream}" +else + export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim}" +fi +# OSMO_AIRSTACK_UP=false skips `airstack up` and goes straight to sleep — the +# pod is fully set up (sshd, dockerd, clone, creds) for the caller to drive. +# The airstack-mission.yaml workflow uses this hook: it runs this entrypoint +# for setup only, then hands off to osmo/workspace/mission_launcher.sh (from +# the clone, so the mission engine ships with the branch — no image rebuild). cd "$AIRSTACK_ROOT" - -if [ -n "${OSMO_MISSION_FILE:-}" ]; then - # Mission mode (airstack-mission.yaml): mission_runner.py owns the full - # stack lifecycle — repeated `airstack down/up` per iteration, PX4 - # readiness gating, mcap recording, and artifact collection into - # /osmo/output. Stack env (NUM_ROBOTS, COMPOSE_PROFILES, - # ISAAC_SIM_SCRIPT_NAME, ...) comes from the mission spec's `env:` block, - # so none of the dev-session defaults below are exported here. - # - # The runner is taken from the clone, not baked into the image, so the - # mission spec and the code executing it always come from the same branch. - MISSION_PATH="$AIRSTACK_ROOT/$OSMO_MISSION_FILE" - [ -f "$MISSION_PATH" ] || MISSION_PATH="$OSMO_MISSION_FILE" # allow absolute paths - if [ ! -f "$MISSION_PATH" ]; then - log "ERROR: mission file not found: $OSMO_MISSION_FILE — pod stays alive for debugging via SSH" - else - log "mission mode: python3 osmo/workspace/mission_runner.py $MISSION_PATH" - if python3 "$AIRSTACK_ROOT/osmo/workspace/mission_runner.py" "$MISSION_PATH" \ - --airstack-root "$AIRSTACK_ROOT"; then - log "mission runner finished: all iterations passed" - else - log "WARN: mission runner exited non-zero — see summary.json / steps.json in the results dir" - fi - fi - if [ "${OSMO_MISSION_KEEP_ALIVE:-true}" != "true" ]; then - log "OSMO_MISSION_KEEP_ALIVE=false — exiting so OSMO uploads /osmo/output and frees the GPU" - # Exit 0 regardless of mission status: a clean task exit is what - # triggers the `outputs:` upload; mission pass/fail lives in summary.json. - exit 0 - fi - log "OSMO_MISSION_KEEP_ALIVE=true — pod stays alive; download results with 'airstack osmo:fetch'" +if [ "${OSMO_AIRSTACK_UP:-true}" = "true" ]; then + log "airstack up (COMPOSE_PROFILES=$COMPOSE_PROFILES, NUM_ROBOTS=$NUM_ROBOTS, livestream=$ISAAC_SIM_LIVESTREAM)" + ./airstack.sh up || log "WARN: airstack up exited non-zero — pod stays alive for debugging via SSH" else - # Dev mode (airstack-dev.yaml): single bring-up, then the student drives. - # Honor optional overrides passed in via OSMO env. Defaults match a - # "single robot, Isaac Sim with WebRTC livestream" dev session. - export AUTOLAUNCH="${AUTOLAUNCH:-true}" - export NUM_ROBOTS="${NUM_ROBOTS:-1}" - export ISAAC_SIM_LIVESTREAM="${ISAAC_SIM_LIVESTREAM:-true}" - - # COMPOSE_PROFILES selection: the default `desktop,isaac-sim` from .env runs - # the standard isaac-sim service. If the student wants livestream, they (or - # we) swap to the isaac-sim-livestream profile, which is the OSMO-friendly - # variant defined in simulation/isaac-sim/docker/docker-compose.yaml. - if [ "$ISAAC_SIM_LIVESTREAM" = "true" ]; then - export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim-livestream}" - else - export COMPOSE_PROFILES="${COMPOSE_PROFILES:-desktop,isaac-sim}" - fi - - if [ "${OSMO_AIRSTACK_UP:-true}" = "true" ]; then - log "airstack up (COMPOSE_PROFILES=$COMPOSE_PROFILES, NUM_ROBOTS=$NUM_ROBOTS, livestream=$ISAAC_SIM_LIVESTREAM)" - ./airstack.sh up || log "WARN: airstack up exited non-zero — pod stays alive for debugging via SSH" - else - log "OSMO_AIRSTACK_UP=false — skipping airstack up; SSH in and run ./airstack.sh up manually" - fi + log "OSMO_AIRSTACK_UP=false — skipping airstack up; caller drives the stack" fi # ─── 7. Sleep ────────────────────────────────────────────────────────────── log "entrypoint complete; sleeping forever so port-forwards keep working" -if [ "${ISAAC_SIM_LIVESTREAM:-false}" = "true" ]; then +if [ "$ISAAC_SIM_LIVESTREAM" = "true" ]; then isaac_sim_log_container="isaac-sim-livestream" else isaac_sim_log_container="airstack-isaac-sim-1" diff --git a/osmo/workspace/mission_launcher.sh b/osmo/workspace/mission_launcher.sh new file mode 100755 index 000000000..a054fb9d8 --- /dev/null +++ b/osmo/workspace/mission_launcher.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# mission_launcher.sh — no-rebuild mission entrypoint for AirStack-on-OSMO. +# +# The airstack-mission.yaml workflow runs the *baked* image entrypoint with +# OSMO_AIRSTACK_UP=false (pod setup only: sshd, inner dockerd, branch clone, +# Nucleus creds, registry login), then hands off to THIS script — which lives +# in the clone, so the mission engine always comes from your branch and no +# workspace-image rebuild is needed to iterate. (PyYAML, which the baked image +# may predate, is installed at runtime below.) +# +# Responsibilities: +# 1. Wait for the setup the baked entrypoint performs in the background +# (inner dockerd ready, clone present, registry login done). +# 2. Ensure PyYAML is importable. +# 3. Run mission_runner.py against $OSMO_MISSION_FILE. +# 4. Keep-alive: stay foreground (sleep) so the pod survives for +# `airstack osmo:fetch`, or exit so OSMO uploads /osmo/output + frees GPU. + +set -uo pipefail + +AIRSTACK_ROOT="${AIRSTACK_ROOT:-/root/AirStack}" +AIRLAB_REGISTRY="${AIRLAB_REGISTRY:-airlab-docker.andrew.cmu.edu}" + +log() { echo "[mission-launcher] $*"; } +fail() { echo "[mission-launcher] ERROR: $*" >&2; } + +# wait_for — poll until the command +# succeeds or the timeout elapses. Returns non-zero on timeout. +wait_for() { + local desc="$1" timeout="$2"; shift 2 + local i=0 + until "$@" >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -ge "$timeout" ]; then + fail "timed out after ${timeout}s waiting for: ${desc}" + return 1 + fi + [ $((i % 15)) -eq 0 ] && log "still waiting for ${desc} (${i}s)" + sleep 1 + done + log "ready: ${desc}" +} + +# ── 1. setup readiness (driven by the backgrounded baked entrypoint) ─────── +# If dockerd or the clone never appear, the pod is broken — stay alive (when +# keep-alive) so it can be inspected over SSH rather than vanishing. +if ! wait_for "inner dockerd" 180 docker info; then + [ "${OSMO_MISSION_KEEP_ALIVE:-true}" = "true" ] && exec sleep infinity + exit 1 +fi +if ! wait_for "branch clone" 600 test -f "$AIRSTACK_ROOT/osmo/workspace/mission_runner.py"; then + [ "${OSMO_MISSION_KEEP_ALIVE:-true}" = "true" ] && exec sleep infinity + exit 1 +fi + +# Registry login is performed by the baked entrypoint (step 5) only when +# AIRLAB_REGISTRY_USER is set. Wait for it so `airstack up`'s image pulls +# don't race the login; warn (don't abort) if it never lands. +if [ -n "${AIRLAB_REGISTRY_USER:-}" ]; then + wait_for "registry login (${AIRLAB_REGISTRY})" 180 \ + grep -q "$AIRLAB_REGISTRY" /root/.docker/config.json \ + || log "WARN: registry login not detected — image pulls may fail" +fi + +# ── 2. PyYAML (mission_runner imports yaml) ──────────────────────────────── +if ! python3 -c "import yaml" >/dev/null 2>&1; then + log "installing PyYAML (image predates python3-yaml)" + pip3 install --break-system-packages --quiet pyyaml \ + || { apt-get update -qq && apt-get install -y -qq python3-yaml; } \ + || fail "could not install PyYAML — mission_runner will fail to import yaml" +fi + +# ── 3. run the mission ───────────────────────────────────────────────────── +MISSION="${OSMO_MISSION_FILE:-}" +if [ -z "$MISSION" ]; then + fail "OSMO_MISSION_FILE not set — nothing to run" +else + MISSION_PATH="$AIRSTACK_ROOT/$MISSION" + [ -f "$MISSION_PATH" ] || MISSION_PATH="$MISSION" # allow an absolute path + if [ -f "$MISSION_PATH" ]; then + log "running mission: $MISSION_PATH" + python3 "$AIRSTACK_ROOT/osmo/workspace/mission_runner.py" "$MISSION_PATH" \ + --airstack-root "$AIRSTACK_ROOT" + log "mission_runner exited $?" + else + fail "mission file not found: $MISSION (looked under the clone and as an absolute path)" + fi +fi + +# ── 4. lifetime ──────────────────────────────────────────────────────────── +if [ "${OSMO_MISSION_KEEP_ALIVE:-true}" = "true" ]; then + log "OSMO_MISSION_KEEP_ALIVE=true — pod stays alive; fetch with 'airstack osmo:fetch'" + exec sleep infinity +fi +log "OSMO_MISSION_KEEP_ALIVE=false — exiting so OSMO uploads /osmo/output and frees the GPU" +exit 0 From 42ecd1c9836d4a9fcb3e4b7b267866eacfca9909 Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Thu, 11 Jun 2026 16:16:10 -0400 Subject: [PATCH 06/27] fixed duplicate airstack up issue and not recording mcap files --- .gitignore | 4 +- osmo/workspace/mission_launcher.sh | 18 ++++++++ osmo/workspace/mission_runner.py | 68 ++++++++++++++++++++++++++---- 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index d14138ebb..624f0d4e0 100644 --- a/.gitignore +++ b/.gitignore @@ -94,8 +94,10 @@ simulation/ms-airsim/assets/scenes/* # Test results tests/results/ -# OSMO mission results (local runs of osmo/workspace/mission_runner.py) +# OSMO mission results: local runs of osmo/workspace/mission_runner.py write +# to osmo/results/; `airstack osmo:fetch` downloads to ./osmo-results/. osmo/results/ +osmo-results/ # Local-only — embedded sibling repo, not part of this branch common/rayfronts/ diff --git a/osmo/workspace/mission_launcher.sh b/osmo/workspace/mission_launcher.sh index a054fb9d8..05c71763a 100755 --- a/osmo/workspace/mission_launcher.sh +++ b/osmo/workspace/mission_launcher.sh @@ -62,6 +62,24 @@ if [ -n "${AIRLAB_REGISTRY_USER:-}" ]; then || log "WARN: registry login not detected — image pulls may fail" fi +# The deployed :latest image's baked entrypoint runs its OWN `airstack up` +# (that image predates the OSMO_AIRSTACK_UP=false hook, so it ignores the +# request to skip it). If the mission started its own bring-up concurrently, +# the two compose runs collide — duplicate network, container-name conflicts, +# "network not found" mid-teardown — and the first iteration fails. So wait +# for the baked entrypoint to FINISH its bring-up (which also warms the inner +# image cache the mission then reuses) before handing off. Detect completion +# by its terminal `sleep infinity`. On a rebuilt image where the hook works, +# the entrypoint skips `up` and reaches that sleep almost immediately, so this +# is a fast no-op — correct either way. +log "waiting for the baked entrypoint to finish its own bring-up (avoids a concurrent 'airstack up')" +sleep 10 # let the baked entrypoint actually reach its `airstack up` +if wait_for "baked entrypoint idle" 2400 pgrep -f "sleep infinity"; then + sleep 5 # let compose fully release the network before the mission's first down/up +else + log "WARN: baked entrypoint still busy after 40m — proceeding; first iteration may race its bring-up" +fi + # ── 2. PyYAML (mission_runner imports yaml) ──────────────────────────────── if ! python3 -c "import yaml" >/dev/null 2>&1; then log "installing PyYAML (image predates python3-yaml)" diff --git a/osmo/workspace/mission_runner.py b/osmo/workspace/mission_runner.py index 74e6aa475..ec662fc28 100644 --- a/osmo/workspace/mission_runner.py +++ b/osmo/workspace/mission_runner.py @@ -184,6 +184,13 @@ def gcs_container(): return names[0] if names else None +def stack_containers(): + """Running containers belonging to the AirStack compose stack (robot, gcs, + and any sim variant — isaac-sim / isaac-sim-livestream / ms-airsim).""" + pats = ("airstack", "isaac-sim", "ms-airsim") + return [n for n in list_containers() if any(p in n for p in pats)] + + # ── mission spec ─────────────────────────────────────────────────────────── def load_mission(path): @@ -304,6 +311,32 @@ def down(self): if result.returncode != 0: log(f"WARN: airstack down exited {result.returncode}") + def ensure_down(self): + """`airstack status`; if any stack containers exist, `airstack down` + and BLOCK until they're actually gone before returning. + + This is the pre-`up` guard: a fresh `airstack up` must not race a + previous bring-up's containers/network (the baked entrypoint leaves a + stack up on a stale image; the prior iteration leaves one too). + Starting `up` while those still exist causes duplicate-network and + container-name conflicts.""" + self._airstack("status", 60) + existing = stack_containers() + if not existing: + return + log(f"existing stack containers found {existing} — bringing them down first") + self.down() + deadline = time.time() + self.mission["down_timeout_s"] + while time.time() < deadline: + remaining = stack_containers() + if not remaining: + log("stack fully down; safe to bring up") + return + log(f"waiting for teardown: {remaining}") + time.sleep(2) + log(f"WARN: containers still present after down: {stack_containers()} " + f"— proceeding, `up` may conflict") + def wait_ready(self, container): """Two sequential gates per robot (same as the system tests): 1. mavros/state reports connected=True (MAVROS ↔ PX4 heartbeat); @@ -419,23 +452,36 @@ def _topic_selection(self, robots): def _start_one(self, container, domain_id, tag, selection, setup_bash): out_dir = f"{BAG_STAGING_DIR}/{tag}" + log_file = f"{BAG_STAGING_DIR}/record_{tag}.log" + pid_file = f"{BAG_STAGING_DIR}/record_{tag}.pid" + # Create the staging dir in the FOREGROUND, THEN background only the + # recorder. Folding `mkdir` into the same `... &` chain backgrounds the + # mkdir too, so the foreground pidfile write races ahead of the dir + # existing ("record_.pid: No such file or directory"). nohup + + # pidfile lets the recorder outlive this docker exec; topics that + # don't exist yet are fine (rosbag2 waits for them). inner = ( - f"{ros2_env_prefix(setup_bash, domain_id)} && " - f"mkdir -p {BAG_STAGING_DIR} && rm -rf {out_dir} && " - # nohup + pidfile: the record process must outlive this docker - # exec; topics that don't exist yet are fine (record waits). + f"mkdir -p {BAG_STAGING_DIR} && rm -rf {out_dir}\n" + f"{ros2_env_prefix(setup_bash, domain_id)}\n" f"nohup ros2 bag record -s mcap -o {out_dir} {selection} " - f"> {BAG_STAGING_DIR}/record_{tag}.log 2>&1 & " - f"echo $! > {BAG_STAGING_DIR}/record_{tag}.pid" + f"> {log_file} 2>&1 &\n" + f"echo $! > {pid_file}\n" + # Confirm the recorder is actually alive — a bad topic name or a + # missing mcap plugin would make it exit immediately, and a silent + # recording failure is the worst outcome for a mission. + f"sleep 2\n" + f"if kill -0 \"$(cat {pid_file})\" 2>/dev/null; then echo RECORDER_ALIVE; " + f"else echo RECORDER_DEAD; tail -n 5 {log_file} 2>/dev/null; fi" ) r = docker_exec(container, inner, timeout=30) - if r.returncode == 0: + if "RECORDER_ALIVE" in r.stdout: self.active.append((container, tag)) n_topics = "all topics" if selection == "-a" else f"{len(selection.split())} topics" log(f"recording [{tag}] in {container} (domain {domain_id}) " f"→ {out_dir} ({n_topics})") else: - log(f"WARN: failed to start recorder [{tag}]: {r.stderr.strip()[:200]}") + log(f"WARN: recorder [{tag}] failed to start / exited immediately: " + f"{tail(r.stdout + r.stderr, 6)}") def start(self): if not self.cfg.get("enabled", True): @@ -719,7 +765,11 @@ def run_iteration(stack, mission, iter_dir): container = None t0 = time.time() try: - stack.down() + # `airstack status`; if a stack is already up (baked entrypoint on a + # stale image, or the previous iteration), down it and wait for the + # containers to fully disappear before bringing up — otherwise the + # `up` races leftover containers/network. + stack.ensure_down() containers = stack.up() container = containers[0] summary["up_duration_s"] = round(time.time() - t0, 2) From 0c582b033504f6447a9f73c1acef1255443d1219 Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Thu, 11 Jun 2026 16:58:32 -0400 Subject: [PATCH 07/27] fixed iteration 1 not working --- osmo/workflows/airstack-mission.yaml | 7 ++++- osmo/workspace/mission_launcher.sh | 44 ++++++++++++---------------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/osmo/workflows/airstack-mission.yaml b/osmo/workflows/airstack-mission.yaml index 80e79a659..7b7498452 100644 --- a/osmo/workflows/airstack-mission.yaml +++ b/osmo/workflows/airstack-mission.yaml @@ -59,7 +59,12 @@ workflow: set -uo pipefail log() { echo "[mission-bootstrap] $*"; } log "starting pod setup via baked entrypoint (OSMO_AIRSTACK_UP=false)" - OSMO_AIRSTACK_UP=false /usr/local/bin/entrypoint.sh & + # Tee the baked entrypoint's output so the launcher can wait for its + # terminal "sleeping forever" line (= setup + its own `airstack up` + # finished) before starting the mission — see mission_launcher.sh. + # The pipe keeps its logs in `osmo workflow logs` too. + OSMO_AIRSTACK_UP=false /usr/local/bin/entrypoint.sh 2>&1 \ + | tee /tmp/baked-entrypoint.log & LAUNCHER=/root/AirStack/osmo/workspace/mission_launcher.sh log "waiting for branch clone to provide $LAUNCHER" for i in $(seq 1 600); do [ -f "$LAUNCHER" ] && break; sleep 2; done diff --git a/osmo/workspace/mission_launcher.sh b/osmo/workspace/mission_launcher.sh index 05c71763a..e7f757123 100755 --- a/osmo/workspace/mission_launcher.sh +++ b/osmo/workspace/mission_launcher.sh @@ -19,7 +19,6 @@ set -uo pipefail AIRSTACK_ROOT="${AIRSTACK_ROOT:-/root/AirStack}" -AIRLAB_REGISTRY="${AIRLAB_REGISTRY:-airlab-docker.andrew.cmu.edu}" log() { echo "[mission-launcher] $*"; } fail() { echo "[mission-launcher] ERROR: $*" >&2; } @@ -53,32 +52,27 @@ if ! wait_for "branch clone" 600 test -f "$AIRSTACK_ROOT/osmo/workspace/mission_ exit 1 fi -# Registry login is performed by the baked entrypoint (step 5) only when -# AIRLAB_REGISTRY_USER is set. Wait for it so `airstack up`'s image pulls -# don't race the login; warn (don't abort) if it never lands. -if [ -n "${AIRLAB_REGISTRY_USER:-}" ]; then - wait_for "registry login (${AIRLAB_REGISTRY})" 180 \ - grep -q "$AIRLAB_REGISTRY" /root/.docker/config.json \ - || log "WARN: registry login not detected — image pulls may fail" -fi - -# The deployed :latest image's baked entrypoint runs its OWN `airstack up` -# (that image predates the OSMO_AIRSTACK_UP=false hook, so it ignores the -# request to skip it). If the mission started its own bring-up concurrently, -# the two compose runs collide — duplicate network, container-name conflicts, -# "network not found" mid-teardown — and the first iteration fails. So wait -# for the baked entrypoint to FINISH its bring-up (which also warms the inner -# image cache the mission then reuses) before handing off. Detect completion -# by its terminal `sleep infinity`. On a rebuilt image where the hook works, -# the entrypoint skips `up` and reaches that sleep almost immediately, so this -# is a fast no-op — correct either way. +# The deployed :latest image's baked entrypoint ignores OSMO_AIRSTACK_UP and +# runs its OWN `airstack up`. Running the mission's bring-up concurrently +# collides (duplicate network, container-name conflicts, "network not found" +# mid-teardown) and fails the first iteration. So wait for the baked +# entrypoint to FINISH — the bootstrap tees its output to $BAKED_LOG, and its +# terminal "sleeping forever" line is printed exactly once, after its +# `airstack up` completes (success or fail). This also subsumes the clone + +# registry-login steps it performs, and warms the inner image cache the +# mission reuses. On a rebuilt image where the hook works, the entrypoint +# skips `up` and prints that line almost immediately, so this is a fast no-op. +# +# We deliberately do NOT detect completion via `pgrep -f "sleep infinity"`: +# the AirStack containers themselves run `sleep infinity`, so that matches the +# instant the baked bring-up starts a container — firing the handoff in the +# middle of its `airstack up`, which is exactly the race we're avoiding. +BAKED_LOG="${BAKED_ENTRYPOINT_LOG:-/tmp/baked-entrypoint.log}" log "waiting for the baked entrypoint to finish its own bring-up (avoids a concurrent 'airstack up')" -sleep 10 # let the baked entrypoint actually reach its `airstack up` -if wait_for "baked entrypoint idle" 2400 pgrep -f "sleep infinity"; then - sleep 5 # let compose fully release the network before the mission's first down/up -else - log "WARN: baked entrypoint still busy after 40m — proceeding; first iteration may race its bring-up" +if ! wait_for "baked entrypoint complete" 2400 grep -q "sleeping forever" "$BAKED_LOG"; then + log "WARN: baked entrypoint didn't signal completion in 40m — proceeding; first iteration may race its bring-up" fi +sleep 5 # let baked's compose settle before the mission's first ensure_down/up # ── 2. PyYAML (mission_runner imports yaml) ──────────────────────────────── if ! python3 -c "import yaml" >/dev/null 2>&1; then From 64594854697ff7013906fabc65bd500bcf1d88d0 Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Wed, 1 Jul 2026 11:45:46 -0400 Subject: [PATCH 08/27] updated git ignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 624f0d4e0..1fb9d8592 100644 --- a/.gitignore +++ b/.gitignore @@ -96,8 +96,8 @@ tests/results/ # OSMO mission results: local runs of osmo/workspace/mission_runner.py write # to osmo/results/; `airstack osmo:fetch` downloads to ./osmo-results/. -osmo/results/ -osmo-results/ +osmo/results/* +osmo-results/* # Local-only — embedded sibling repo, not part of this branch common/rayfronts/ From 3ab223f50d125020a895a6be8181c763c7f87e8b Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Wed, 15 Jul 2026 14:54:21 -0400 Subject: [PATCH 09/27] updated with airlab info --- .airstack/modules/osmo.sh | 236 +++++- .env | 2 +- osmo/missions/README.md | 42 +- osmo/missions/example_takeoff_land.yaml | 9 +- osmo/workflows/airstack-mission.yaml | 31 +- osmo/workspace/Dockerfile | 1 + osmo/workspace/mission_launcher.sh | 57 ++ osmo/workspace/mission_runner.py | 723 ++++++++++++++++-- .../isaac-sim/extensions/PegasusSimulator | 2 +- 9 files changed, 1028 insertions(+), 75 deletions(-) diff --git a/.airstack/modules/osmo.sh b/.airstack/modules/osmo.sh index a10a47809..66ac29e02 100755 --- a/.airstack/modules/osmo.sh +++ b/.airstack/modules/osmo.sh @@ -120,19 +120,47 @@ function _osmo_prompt { fi } -# osmo:setup — interactively register the three OSMO credentials AirStack -# needs (airlab-docker-registry, airlab-docker-login, airlab-nucleus). -# Idempotent — re-running rotates the credentials. +# Helper: _osmo_prompt variant that falls back to $3 on empty input. +function _osmo_prompt_default { + local var_name="$1" + local prompt_text="$2" + local default_val="$3" + local saved_stty="" + + if [ -t 0 ]; then + saved_stty="$(stty -g 2>/dev/null || true)" + if [ -n "$saved_stty" ]; then + trap 'stty "$saved_stty" 2>/dev/null; trap - INT' INT + stty -icanon 2>/dev/null + fi + fi + read -r -p "${prompt_text} [${default_val}]: " "$var_name" + if [ -n "$saved_stty" ]; then + stty "$saved_stty" 2>/dev/null + trap - INT + fi + + _osmo_trim "$var_name" + if [ -z "${!var_name}" ]; then + printf -v "$var_name" '%s' "$default_val" + fi +} + +# osmo:setup — interactively register the OSMO credentials AirStack needs +# (airlab-docker-registry, airlab-docker-login, airlab-nucleus, optional +# airlab-storage). Idempotent — re-running rotates the credentials. function cmd_osmo_setup { _osmo_check_cli || return 1 cat >&2 <<'EOF' -This sets up the three per-user OSMO credentials AirStack-on-OSMO needs: +This sets up the per-user OSMO credentials AirStack-on-OSMO needs: 1. airlab-docker-registry (REGISTRY) — for OSMO to pull the workspace image 2. airlab-docker-login (GENERIC) — for the inner dockerd to pull AirStack images 3. airlab-nucleus (GENERIC) — for Isaac Sim Nucleus access + 4. airlab-storage (GENERIC) — OPTIONAL: push mission results to the + airlab-storage NAS + auto-teardown You'll be asked for: @@ -140,6 +168,9 @@ You'll be asked for: - your AirLab Docker password (same as your Andrew password) - your Nucleus API token (https://airlab-nucleus.andrew.cmu.edu/omni/web3/ → right-click cloud → API Tokens). NOT your Andrew password. + - (optional) your airlab-storage password — the SEPARATE Samba/SFTP/rsync + password set at https://airlab-storage.andrew.cmu.edu:4001/#/forgot-pwd, + NOT your Andrew/Docker password. Values go directly to OSMO; nothing is written to disk locally. @@ -202,7 +233,33 @@ EOF "omni_server=${omni_server}" \ || { log_error "osmo credential set airlab-nucleus failed"; return 1; } - log_info "All three credentials registered. List them with: osmo credential list" + # Optional: airlab-storage NAS upload. Only the password is a secret; the + # destination path is set per-mission (nas_dest: in the mission spec). + local want_storage="" + read -r -p "Configure airlab-storage NAS upload (optional)? [y/N]: " want_storage + case "$want_storage" in + y|Y|yes|Yes) + local storage_user storage_pass storage_host + _osmo_prompt_default storage_user "airlab-storage username (Andrew ID)" "$andrew_id" + _osmo_prompt storage_pass "airlab-storage password (hidden; the SEPARATE Samba/SFTP/rsync password)" true || return 1 + _osmo_prompt_default storage_host "airlab-storage host" "airlab-storage.andrew.cmu.edu" + + log_info "Refreshing airlab-storage (GENERIC)..." + osmo credential delete airlab-storage >/dev/null 2>&1 || true + osmo credential set airlab-storage \ + --type GENERIC \ + --payload "username=${storage_user}" \ + "password=${storage_pass}" \ + "host=${storage_host}" \ + || { log_error "osmo credential set airlab-storage failed"; return 1; } + log_info "airlab-storage registered. Set 'nas_dest: /volume//...' in a mission spec to upload results + auto-teardown." + ;; + *) + log_info "Skipping airlab-storage — NAS upload stays disabled (missions keep-alive as before)." + ;; + esac + + log_info "Credentials registered. List them with: osmo credential list" log_info "Next: airstack osmo:up [--pool POOL]" } @@ -686,16 +743,92 @@ function cmd_osmo_foxglove { --connect-timeout "$OSMO_PF_TIMEOUT" } +# Parse a duration (90s / 30m / 8h / bare number = minutes) to seconds. +function _osmo_parse_duration { + local v="$1" + if [[ "$v" =~ ^([0-9]+)([smh]?)$ ]]; then + local n="${BASH_REMATCH[1]}" u="${BASH_REMATCH[2]}" + case "$u" in + s) echo "$n" ;; + h) echo $(( n * 3600 )) ;; + *) echo $(( n * 60 )) ;; # m or bare → minutes + esac + else + log_error "invalid duration '$v' — use e.g. 8h, 480m, 3600s, or a bare number (minutes)" + return 1 + fi +} + +function _osmo_hms { + printf '%dh%02dm' $(( $1 / 3600 )) $(( ($1 % 3600) / 60 )) +} + +# Block until the mission finishes (the workspace log prints "mission_runner +# exited") or cap_s elapses, then run osmo:fetch. Requires keep-alive so the +# pod is still up to fetch from. +function _osmo_wait_and_fetch { + local wf="$1" cap_s="$2" + local dest="${OSMO_AUTOFETCH_DEST:-./osmo-results}" + local poll="${OSMO_AUTOFETCH_POLL_S:-120}" + local start now elapsed snapshot status + start="$(date +%s)" + log_info "auto-fetch armed: polling every ${poll}s for completion (cap $(_osmo_hms "$cap_s")) → ${dest}" + log_info " keep this terminal open (or background with nohup/tmux) until the fetch runs." + while :; do + now="$(date +%s)"; elapsed=$(( now - start )) + status="$(osmo workflow query "$wf" 2>/dev/null | awk -F': +' '/^Status/ {print $2; exit}' | tr -d ' \r\n' || true)" + case "$status" in + PENDING|RUNNING|"") ;; + *) log_warn "workflow ${wf} is ${status} — ending wait, attempting fetch."; break ;; + esac + snapshot="$(timeout 25 osmo workflow logs "$wf" -t workspace -n 600 2>/dev/null || true)" + if printf '%s\n' "$snapshot" | grep -q "mission_runner exited"; then + log_info "mission complete (after $(_osmo_hms "$elapsed")) — fetching." + break + fi + if [ "$elapsed" -ge "$cap_s" ]; then + log_warn "auto-fetch cap reached ($(_osmo_hms "$cap_s")) without completion — fetching what's there." + break + fi + sleep "$poll" + done + AIRSTACK_OSMO_WF="$wf" cmd_osmo_fetch "$dest" +} + +# osmo:autofetch — attach the auto-fetch poller to an already-running mission +# (one submitted without --auto-fetch, or whose terminal was closed). Polls the +# workspace log for completion, then runs osmo:fetch. +# +# Usage: airstack osmo:autofetch [DUR] (DUR = max-wait cap, default 8h; uses +# the saved workflow id, or AIRSTACK_OSMO_WF to target a specific one.) +function cmd_osmo_autofetch { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + local cap_s + cap_s="$(_osmo_parse_duration "${1:-8h}")" || return 1 + _osmo_wait_and_fetch "$wf" "$cap_s" +} + # osmo:mission — submit airstack-mission.yaml with a mission spec selected. # # Usage: airstack osmo:mission [--pool POOL] [--key PATH] # [--branch BRANCH] [--no-keep-alive] +# [--auto-fetch DUR] [--nas-dest PATH] +# [--no-nas-upload] # # is a repo-relative path (e.g. osmo/missions/example_takeoff_land.yaml). # The pod clones the branch and runs the mission spec from that clone, so the # mission file must be committed and pushed. --no-keep-alive makes the task # exit when the mission ends (frees the GPU, triggers the workflow's # `outputs:` upload) instead of sleeping for `airstack osmo:fetch`. +# +# --auto-fetch DUR blocks after submit, polling the workspace log for mission +# completion and running osmo:fetch as soon as it's done (DUR — e.g. 8h, 480m — +# is the max wait before fetching anyway). Requires keep-alive (the default). +# +# If the airlab-storage credential is set (airstack osmo:setup) and the mission +# spec has `nas_dest:` (or --nas-dest PATH overrides it), the pod rsyncs results +# to the NAS then tears itself down. --no-nas-upload suppresses that. function cmd_osmo_mission { _osmo_check_cli || return 1 @@ -705,6 +838,9 @@ function cmd_osmo_mission { local branch="" local branch_explicit=false local keep_alive="true" + local auto_fetch_dur="" + local nas_dest="" + local no_nas_upload="false" local extra_args=() while [ $# -gt 0 ]; do @@ -713,6 +849,9 @@ function cmd_osmo_mission { --key) pubkey_file="$2"; shift 2 ;; --branch) branch="$2"; branch_explicit=true; shift 2 ;; --no-keep-alive) keep_alive="false"; shift ;; + --auto-fetch) auto_fetch_dur="$2"; shift 2 ;; + --nas-dest) nas_dest="$2"; shift 2 ;; + --no-nas-upload) no_nas_upload="true"; shift ;; -*) extra_args+=("$1"); shift ;; *) if [ -z "$mission" ]; then mission="$1"; else extra_args+=("$1"); fi @@ -720,6 +859,23 @@ function cmd_osmo_mission { esac done + # Validate --auto-fetch up front (fail before submitting on a bad value). + local auto_fetch_cap_s="" + if [ -n "$auto_fetch_dur" ]; then + if [ "$keep_alive" != "true" ]; then + log_error "--auto-fetch requires the pod to stay alive; don't combine it with --no-keep-alive." + return 1 + fi + auto_fetch_cap_s="$(_osmo_parse_duration "$auto_fetch_dur")" || return 1 + fi + + # NAS auto-upload tears the pod down on completion, so --auto-fetch (which + # needs the pod alive) is moot when it runs. Best-effort heads-up. + if [ -n "$auto_fetch_dur" ] && [ "$no_nas_upload" != "true" ] \ + && osmo credential list 2>/dev/null | grep -q 'airlab-storage'; then + log_warn "airlab-storage is configured; a mission with nas_dest set uploads + tears down, making --auto-fetch moot." + fi + if [ -z "$mission" ]; then log_error "Usage: airstack osmo:mission [--pool POOL] [--branch BRANCH] [--no-keep-alive]" log_error "Available missions:" @@ -777,6 +933,12 @@ function cmd_osmo_mission { if [ -n "$branch" ]; then env_kvs+=("AIRSTACK_BRANCH=${branch}") fi + if [ -n "$nas_dest" ]; then + env_kvs+=("OSMO_MISSION_UPLOAD_DEST=${nas_dest}") + fi + if [ "$no_nas_upload" = "true" ]; then + env_kvs+=("OSMO_MISSION_NO_UPLOAD=true") + fi cmd+=(--set-env "${env_kvs[@]}") if [ ${#extra_args[@]} -gt 0 ]; then cmd+=("${extra_args[@]}") @@ -800,10 +962,18 @@ function cmd_osmo_mission { fi _osmo_save_wf_id "$wf_id" + if [ -n "$auto_fetch_cap_s" ]; then + _osmo_wait_and_fetch "$wf_id" "$auto_fetch_cap_s" + return $? + fi + log_info "Next steps:" log_info " airstack osmo:logs # follow mission progress" log_info " airstack osmo:fetch # download bags + results (keep-alive mode)" log_info " airstack osmo:down # cancel when done (results die with the pod!)" + if [ -n "$nas_dest" ] && [ "$no_nas_upload" != "true" ]; then + log_info "NAS upload armed → results go to ${nas_dest}///, then the pod tears itself down." + fi } # osmo:fetch — download mission results (mcap bags, logs, summaries) from the @@ -879,6 +1049,54 @@ function cmd_osmo_fetch { log_info "Done. Open any .mcap under ${dest} directly in Foxglove (Open local file)." } +# osmo:stop — gracefully stop the running mission: SIGINT to mission_runner +# on the pod → current step aborts, recorders finalize their mcaps, bags + +# logs are collected, stack goes down. The pod itself stays alive +# (keep-alive), so follow with `airstack osmo:fetch` to download results. +function cmd_osmo_stop { + _osmo_check_cli || return 1 + local wf; wf="$(_osmo_wf_id)" || return 1 + + local local_port="${OSMO_SSH_PORT%%:*}" + local ssh_opts=(-p "$local_port" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR) + + # Reuse an existing ssh port-forward or spawn one for this command — + # same pattern as osmo:fetch. + local pf_pid="" + if ! nc -z localhost "$local_port" 2>/dev/null; then + log_info "osmo workflow port-forward ${wf} workspace --port ${OSMO_SSH_PORT} (for the stop)" + osmo workflow port-forward "$wf" workspace --port "$OSMO_SSH_PORT" --connect-timeout 600 \ + > "${OSMO_STATE_DIR}/stop-pf.log" 2>&1 & + pf_pid=$! + local waited=0 + until nc -z localhost "$local_port" 2>/dev/null; do + sleep 1; waited=$((waited+1)) + if [ "$waited" -ge 30 ]; then + log_error "Timed out waiting for port-forward on :${local_port}. Log: ${OSMO_STATE_DIR}/stop-pf.log" + [ -n "$pf_pid" ] && kill "$pf_pid" 2>/dev/null + return 1 + fi + done + fi + + log_info "Sending stop signal to mission_runner on the pod..." + ssh "${ssh_opts[@]}" root@localhost \ + "pkill -INT -f 'osmo/workspace/mission_runner\.py' \ + && echo 'stop signal delivered' \ + || echo 'no running mission_runner found (mission already finished?)'" + local rc=$? + + if [ -n "$pf_pid" ]; then + kill "$pf_pid" 2>/dev/null + fi + if [ "$rc" -ne 0 ]; then + log_error "ssh to pod failed (exit ${rc})." + return 1 + fi + log_info "Mission stopping: current step aborts, recordings finalize, bags + logs are collected, stack goes down." + log_info "Watch progress: airstack osmo:logs Download results: airstack osmo:fetch" +} + # osmo:down — cancel the active workflow. Reminds you to push first. function cmd_osmo_down { _osmo_check_cli || return 1 @@ -899,16 +1117,20 @@ function register_osmo_commands { COMMANDS["osmo:up"]="cmd_osmo_up" COMMANDS["osmo:mission"]="cmd_osmo_mission" COMMANDS["osmo:fetch"]="cmd_osmo_fetch" + COMMANDS["osmo:autofetch"]="cmd_osmo_autofetch" + COMMANDS["osmo:stop"]="cmd_osmo_stop" COMMANDS["osmo:logs"]="cmd_osmo_logs" COMMANDS["osmo:ide"]="cmd_osmo_ide" COMMANDS["osmo:webrtc"]="cmd_osmo_webrtc" COMMANDS["osmo:foxglove"]="cmd_osmo_foxglove" COMMANDS["osmo:down"]="cmd_osmo_down" - COMMAND_HELP["osmo:setup"]="One-time per-user OSMO credential setup (airlab-docker-registry, airlab-docker-login, airlab-nucleus)" + COMMAND_HELP["osmo:setup"]="One-time per-user OSMO credential setup (airlab-docker-registry, airlab-docker-login, airlab-nucleus, optional airlab-storage for NAS upload)" COMMAND_HELP["osmo:up"]="Submit osmo/workflows/airstack-dev.yaml with your SSH pubkey injected (--pool POOL, --key PATH, --branch BRANCH)" - COMMAND_HELP["osmo:mission"]="Submit a batch mission (osmo/missions/*.yaml): repeated up→fly→record→down cycles (--pool POOL, --branch BRANCH, --no-keep-alive)" + COMMAND_HELP["osmo:mission"]="Submit a batch mission (osmo/missions/*.yaml): repeated up→fly→record→down cycles (--pool POOL, --branch BRANCH, --no-keep-alive, --auto-fetch DUR, --nas-dest PATH, --no-nas-upload)" COMMAND_HELP["osmo:fetch"]="Download mission results (mcap bags, logs, summaries) from the pod over ssh — incremental, safe to run mid-mission (osmo:fetch [dest-dir])" + COMMAND_HELP["osmo:autofetch"]="Attach the auto-fetch poller to the running mission (saved id or AIRSTACK_OSMO_WF): poll the workspace log until done, then osmo:fetch (osmo:autofetch [DUR], default 8h)" + COMMAND_HELP["osmo:stop"]="Gracefully stop the running mission: abort current step, finalize mcaps, collect bags+logs, stack down (pod stays up for osmo:fetch)" COMMAND_HELP["osmo:logs"]="Follow the workspace task logs (osmo workflow logs -t workspace -n 500; OSMO_LOGS_TASK / OSMO_LOGS_TAIL override)" COMMAND_HELP["osmo:ide"]="Port-forward sshd (2200:22) and open VS Code/Cursor on Host airstack-osmo" COMMAND_HELP["osmo:webrtc"]="Port-forward Isaac Sim WebRTC ranges (TCP foreground + UDP background)" diff --git a/.env b/.env index 412801cb6..70138406b 100644 --- a/.env +++ b/.env @@ -12,7 +12,7 @@ PROJECT_NAME="airstack" # If you've run ./airstack.sh setup, then this will auto-generate from the git commit hash every time a change is made # to a Dockerfile or docker-compose.yaml file. Otherwise this can also be set explicitly to make a release version. # auto-generated from git commit hash -VERSION="8b927e46" +VERSION="64594854" # Choose "dev" or "prebuilt". "dev" is for mounted code that must be built live. "prebuilt" is for built ros_ws baked into the image DOCKER_IMAGE_BUILD_MODE="dev" # Where to push and pull images from. Can replace with your docker hub username if using docker hub. diff --git a/osmo/missions/README.md b/osmo/missions/README.md index d0d95823d..f33b977e7 100644 --- a/osmo/missions/README.md +++ b/osmo/missions/README.md @@ -36,6 +36,7 @@ osmo/results/// └── iter_001/ ├── bags/robot_1/*.mcap # open directly in Foxglove (no conversion) ├── logs/.log # docker logs snapshot per container + ├── logs// # raw /tmp stdout tees (gossip, ddsrouter, relay) + rcl control-stack logs ├── ready.json # per-robot seconds-to-PX4-ready ├── steps.json # per-step command, output tail, pass/fail └── iteration.json # iteration summary @@ -57,16 +58,22 @@ Top-level keys (everything except `steps` is optional): |---|---|---| | `name` | filename stem | Results directory name | | `env` | `{}` | Env vars exported before each `airstack up` (`NUM_ROBOTS`, `COMPOSE_PROFILES`, `ISAAC_SIM_SCRIPT_NAME`, …) | -| `iterations` | `1` | Number of full up→fly→down cycles | +| `iterations` | `1` | Full up→fly→down cycles. With `environments:` + `environment_order: grouped`, this is **per environment** (total = iterations × #environments) | +| `iteration_attempts` | `1` | Max times to (re)run a single iteration until it passes. `>1`: a failed/errored iteration is redone (clean down→up→fly→down) up to this many attempts; a `passed`, manually `stopped`, or `abort_mission` outcome is never retried. Failed attempts' artifacts are preserved under `iter_NNN_failed_attempt_K`. With a fixed `SPAWN_SEED` the redo reproduces the same spawn layout | +| `environment_order` | `round_robin` | With `environments:`: `round_robin` (iteration i → env[(i-1) % n], `iterations` is the total) \| `grouped` (each env runs `iterations` times in a row) | | `ready.timeout_s` | `600` | Max seconds to wait for PX4 readiness per iteration | | `ready.poll_interval_s` | `5` | Seconds between readiness polls | | `record.enabled` | `true` | Record an mcap per robot per iteration | +| `record.scope` | `gcs` | `gcs` (one mcap on GCS domain 0) \| `robot` (one mcap per robot domain) \| `both` | | `record.topics` | tf + odom set | Topics to record; `{robot}` → `robot_N` | | `record.all` | `false` | Record **all** topics (`ros2 bag record -a`) — large | +| `record.exclude` | — | With `all`: regex of topics to drop (`ros2 bag record -a --exclude-regex `) | +| `record.required` | `false` | Abort the iteration if any recorder fails to start (don't fly unrecorded) | | `on_step_failure` | `abort_iteration` | `continue` \| `abort_iteration` \| `abort_mission` | | `up_timeout_s` | `3600` | `airstack up` timeout (first up on a fresh pod pulls images) | | `down_timeout_s` | `300` | `airstack down` timeout | | `robot_setup_bash` | robot ws `setup.bash` | Workspace sourced before `ros2` commands | +| `nas_dest` | — | Base path on airlab-storage (e.g. `/volume3//airstack-missions`). When the `airlab-storage` OSMO credential is set (`airstack osmo:setup`), an OSMO pod rsyncs results to `///` then tears itself down. Override per run with `osmo:mission --nas-dest PATH`; suppress with `--no-nas-upload`. Credentials never live in the spec | | `steps` | — | Ordered list of steps (below) | ### Steps @@ -82,11 +89,37 @@ result. The step passes when the action result reports `success: true`. - action: task: takeoff # → /robot_N/tasks/ goal: {target_altitude_m: 10.0, velocity_m_s: 1.0} - timeout_s: 120 # default 120 + timeout_s: 120 # default 120 (per attempt) + attempts: 3 # per-robot retries on failure (default 3) + retry_delay_s: 10 # wait between attempts (default 10) + feedback_timeout_s: 15 # via gcs: no relay_feedback within this + # window ⇒ goal presumed lost, retried + pass_on_feedback: false # via gcs: pass as soon as feedback is seen + # (the task RAN), regardless of success/fail; + # only "never ran" (no feedback + no result + # after retries) fails the step robots: all # type: task_msgs/action/TakeoffTask # derived from task name if omitted ``` +Any step may also carry **`optional: true`** (a sibling key, not inside +`action`/`run`/…): the step still runs and its result is recorded, but a +failure neither trips `on_step_failure` nor counts toward an `iteration_attempts` +redo. Use it for steps whose outcome doesn't gate the iteration — e.g. a `land` +after the run is already done. + +**`pass_on_feedback` + `iteration_attempts` idiom** — to guarantee a task +*runs* every iteration without caring whether it succeeds: set +`pass_on_feedback: true` on that action and `iteration_attempts: >1` on the +mission. If the task never gets feedback (goal lost / never started) after its +`attempts` retries, the step fails and the whole iteration is redone; once +feedback is seen the iteration is considered satisfied. + +Each robot's goal is logged per attempt and retried independently — a goal +can be rejected transiently (relay has no GPS fix yet, PX4 position estimate +not converged, action server still starting), so one robot failing its first +attempt doesn't fail the step unless it exhausts all attempts. + Available tasks (action type is derived as `task_msgs/action/Task`): `takeoff`, `land`, `fixed_trajectory`, `navigate`, `exploration`, `coverage`, `semantic_search`, `chat`. Goal fields are defined in @@ -101,7 +134,8 @@ Multi-robot action goals are sent **in parallel** across robots. **`run`** — arbitrary command; the escape hatch that makes any ROS 2 command work without runner changes. The step fails on non-zero exit unless -`expect_success: false`. +`expect_success: false`. Set `attempts` to retry on failure (e.g. a flaky +model download) before the step is marked failed. ```yaml - run: @@ -112,6 +146,8 @@ work without runner changes. The step fails on non-zero exit unless cmd: ros2 topic echo --once /{robot}/odometry timeout_s: 60 expect_success: true + attempts: 1 # retries on failure (default 1 = no retry) + retry_delay_s: 10 # wait between attempts (default 10) ``` **`topic_pub`** — `ros2 topic pub --once` per robot: diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml index 1eaa96554..e64bf7892 100644 --- a/osmo/missions/example_takeoff_land.yaml +++ b/osmo/missions/example_takeoff_land.yaml @@ -11,6 +11,12 @@ name: example_takeoff_land +# Optional: upload results to airlab-storage, then auto-teardown once the +# upload succeeds (needs the airlab-storage OSMO credential — see +# `airstack osmo:setup`). Override the path with osmo:mission --nas-dest, +# or suppress with --no-nas-upload. +# nas_dest: /volume//airstack-missions + # How to bring the stack up. `services` are passed straight through: # ./airstack.sh up isaac-sim robot-desktop gcs # (docker compose auto-enables a named service's profile, so no @@ -78,7 +84,8 @@ on_step_failure: abort_iteration # robot selection — `robots:` (default `all` = robots 1..NUM_ROBOTS, or a # list like [1, 3]) — and {robot} → robot_N / {n} → N expand per robot. # Per-step knobs (all optional, shown here with their defaults): -# action: robots=all via= timeout_s=120 +# action: robots=all via= timeout_s=120 (per attempt) +# attempts=3 retry_delay_s=10 (per-robot retries on failure) # via: gcs (through the GCS action_relay) | robot (direct send_goal) # run: robots=all container=robot_{n} (per-robot) or robot_1 | gcs | pod | # ; fans out when cmd/container references {robot}/{n} diff --git a/osmo/workflows/airstack-mission.yaml b/osmo/workflows/airstack-mission.yaml index 7b7498452..7429cae27 100644 --- a/osmo/workflows/airstack-mission.yaml +++ b/osmo/workflows/airstack-mission.yaml @@ -38,6 +38,10 @@ # Set OSMO_MISSION_KEEP_ALIVE=false for fire-and-forget runs: the task exits # cleanly when the mission ends, which frees the GPU and triggers the # `outputs:` upload below (if a destination bucket is configured). +# +# NAS upload: if the airlab-storage credential is set (airstack osmo:setup) and +# the mission spec has `nas_dest:`, the pod rsyncs results to airlab-storage +# then tears itself down — no laptop fetch needed. See mission_launcher.sh. workflow: name: airstack-mission @@ -59,12 +63,15 @@ workflow: set -uo pipefail log() { echo "[mission-bootstrap] $*"; } log "starting pod setup via baked entrypoint (OSMO_AIRSTACK_UP=false)" - # Tee the baked entrypoint's output so the launcher can wait for its - # terminal "sleeping forever" line (= setup + its own `airstack up` - # finished) before starting the mission — see mission_launcher.sh. - # The pipe keeps its logs in `osmo workflow logs` too. + # Baked entrypoint for pod setup, backgrounded. `tee` streams its logs to + # `osmo workflow logs` and to a file the launcher greps for the terminal + # "sleeping forever" line. `tee` holds THIS task's stdout pipe open, so + # mission_launcher.sh kills BAKED_PID before it exits — otherwise + # osmo_exec never sees pipe EOF on our exit and the pod never frees the + # GPU (it hangs at RUNNING until exec_timeout). OSMO_AIRSTACK_UP=false /usr/local/bin/entrypoint.sh 2>&1 \ | tee /tmp/baked-entrypoint.log & + export BAKED_PID=$! LAUNCHER=/root/AirStack/osmo/workspace/mission_launcher.sh log "waiting for branch clone to provide $LAUNCHER" for i in $(seq 1 600); do [ -f "$LAUNCHER" ] && break; sleep 2; done @@ -97,6 +104,8 @@ workflow: credentials: # Same per-user credentials as airstack-dev.yaml — see # docs/tutorials/airstack_on_osmo.md "Step 0" / `airstack osmo:setup`. + # These map OSMO credential fields → pod env vars; no secret values live + # in this file. airlab-nucleus: OMNI_USER: omni_user OMNI_PASS: omni_pass @@ -104,6 +113,12 @@ workflow: airlab-docker-login: AIRLAB_REGISTRY_USER: username AIRLAB_REGISTRY_PASS: password + # Optional: NAS result upload. Absent credential → nothing injected → + # upload disabled. Enabled per-mission via `nas_dest:` in the spec. + airlab-storage: + AIRLAB_STORAGE_USER: username + AIRLAB_STORAGE_PASS: password + AIRLAB_STORAGE_HOST: host resources: default: @@ -117,7 +132,7 @@ workflow: storage: 500Gi timeout: - # Missions run unattended and multi-iteration; 24h leaves room for long - # batches plus result download time in keep-alive mode. The pod is gone - # when this expires — fetch results first. - exec_timeout: 24h + # With keep-alive the pod sleeps after the mission until this expires (then + # it's gone — fetch first). Sized to stay up ~30h post-mission for + # osmo:fetch (mission itself runs in the first ~1-2h). + exec_timeout: 32h diff --git a/osmo/workspace/Dockerfile b/osmo/workspace/Dockerfile index 0e10d9e49..d6a43f5d1 100644 --- a/osmo/workspace/Dockerfile +++ b/osmo/workspace/Dockerfile @@ -44,6 +44,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3-pip \ python3-yaml \ rsync \ + sshpass \ sudo \ tmux \ tzdata \ diff --git a/osmo/workspace/mission_launcher.sh b/osmo/workspace/mission_launcher.sh index e7f757123..b470d10c7 100755 --- a/osmo/workspace/mission_launcher.sh +++ b/osmo/workspace/mission_launcher.sh @@ -23,6 +23,18 @@ AIRSTACK_ROOT="${AIRSTACK_ROOT:-/root/AirStack}" log() { echo "[mission-launcher] $*"; } fail() { echo "[mission-launcher] ERROR: $*" >&2; } +# Kill the backgrounded `tee` (BAKED_PID, from the workflow bootstrap) so it +# releases this task's OSMO stdout pipe — else osmo_exec never sees EOF on our +# exit and the pod never frees the GPU. Call right before an intentional exit. +release_osmo_pipe() { [ -n "${BAKED_PID:-}" ] && kill "$BAKED_PID" 2>/dev/null; true; } + +# Fast DDS LARGE_DATA uses shared memory between same-host/same-version +# participants; the segments live in /dev/shm, which every stack container +# bind-mounts from the pod. The pod default is 64M — it fills immediately +# and later participants then fail SHM registration. Regrow in place. +mount -o remount,size=8G /dev/shm 2>/dev/null +log "/dev/shm: $(df -h /dev/shm | awk 'NR==2{print $2" total, "$5" used"}')" + # wait_for — poll until the command # succeeds or the timeout elapses. Returns non-zero on timeout. wait_for() { @@ -99,10 +111,55 @@ else fi fi +# ── 3b. NAS upload + teardown (optional) ─────────────────────────────────── +# With the airlab-storage credential injected and a destination set (--nas-dest, +# else the spec's nas_dest:), rsync results to the NAS. Success → pod exits (GPU +# freed); failure → stay alive so results survive for osmo:fetch. +NAS_YAML_DEST="" +if [ -f "${MISSION_PATH:-}" ]; then + NAS_YAML_DEST="$(python3 -c 'import yaml,sys; d=yaml.safe_load(open(sys.argv[1])) or {}; print(d.get("nas_dest") or "")' "$MISSION_PATH" 2>/dev/null)" +fi +NAS_DEST="${OSMO_MISSION_UPLOAD_DEST:-$NAS_YAML_DEST}" + +if [ "${OSMO_MISSION_NO_UPLOAD:-false}" != "true" ] \ + && [ -n "${AIRLAB_STORAGE_USER:-}" ] && [ -n "${AIRLAB_STORAGE_PASS:-}" ] \ + && [ -n "$NAS_DEST" ]; then + if ! command -v sshpass >/dev/null 2>&1; then + log "installing sshpass (image predates it)" + { apt-get update -qq && apt-get install -y -qq sshpass; } \ + || fail "could not install sshpass — skipping NAS upload" + fi + if command -v sshpass >/dev/null 2>&1; then + nas_host="${AIRLAB_STORAGE_HOST:-airlab-storage.andrew.cmu.edu}" + results_dir="${OSMO_RESULTS_ROOT:-$AIRSTACK_ROOT/osmo/results}" + ssh_cmd="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" + log "NAS upload: ${results_dir}/ → ${nas_host}:${NAS_DEST%/}/" + export SSHPASS="$AIRLAB_STORAGE_PASS" + # NAS share (Synology rsync 3.1.x) rejects --mkpath and can't chmod/chown the + # SMB-backed volume, so skip those attrs; nas_dest base must already exist. + sshpass -e rsync -rltz --partial --timeout=1800 \ + --no-perms --no-owner --no-group --omit-dir-times -e "$ssh_cmd" \ + "${results_dir}/" "${AIRLAB_STORAGE_USER}@${nas_host}:${NAS_DEST%/}/" + nas_rc=$? + unset SSHPASS + if [ "$nas_rc" -eq 0 ]; then + log "NAS upload OK → ${nas_host}:${NAS_DEST} — tearing pod down (GPU freed)" + release_osmo_pipe + exit 0 + fi + fail "NAS upload FAILED (rc=$nas_rc) — keeping pod alive for 'airstack osmo:fetch' / SSH debug" + exec sleep infinity + fi +elif [ "${OSMO_MISSION_NO_UPLOAD:-false}" != "true" ] \ + && [ -n "${AIRLAB_STORAGE_USER:-}" ] && [ -z "$NAS_DEST" ]; then + log "airlab-storage set but no nas_dest / --nas-dest — skipping upload; pod stays alive." +fi + # ── 4. lifetime ──────────────────────────────────────────────────────────── if [ "${OSMO_MISSION_KEEP_ALIVE:-true}" = "true" ]; then log "OSMO_MISSION_KEEP_ALIVE=true — pod stays alive; fetch with 'airstack osmo:fetch'" exec sleep infinity fi log "OSMO_MISSION_KEEP_ALIVE=false — exiting so OSMO uploads /osmo/output and frees the GPU" +release_osmo_pipe exit 0 diff --git a/osmo/workspace/mission_runner.py b/osmo/workspace/mission_runner.py index ec662fc28..110b8067d 100644 --- a/osmo/workspace/mission_runner.py +++ b/osmo/workspace/mission_runner.py @@ -12,8 +12,9 @@ Artifacts land under one results root per mission run: /// - ├── summary.json # per-iteration pass/fail + durations - └── iter_001/ + ├── summary.json # per-iteration pass/fail + durations + method + └── iter_001____/ # e.g. iter_001__warehouse__frontier + # (iter_NNN prefix kept; env+method appended) ├── bags/robot_1/*.mcap # Foxglove-ready (open the .mcap directly) ├── logs/.log # docker logs snapshot, per container ├── ready.json # per-robot PX4 readiness timings @@ -37,9 +38,12 @@ import argparse import json import os +import re import shlex +import signal import subprocess import sys +import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone @@ -94,11 +98,26 @@ MISSION_DEFAULTS = { "iterations": 1, + # How many times to (re)run a single iteration that doesn't pass before + # giving up and moving on. 1 = no redo (the historical behavior). >1 = if an + # iteration ends failed/errored (a step up to and including the gating step + # failed — e.g. takeoff never armed, or semantic_search never got feedback), + # tear the stack down and run that same iteration again, up to this many + # total attempts. A `stopped` (manual) or `abort_mission` outcome is never + # retried. Failed attempts' artifacts are preserved under + # iter_NNN_failed_attempt_K so the canonical iter_NNN holds the last attempt. + "iteration_attempts": 1, + # With an `environments:` list: round_robin → iteration i uses + # environments[(i-1) % n], so `iterations` is the TOTAL run count. + # grouped → each environment runs `iterations` times consecutively + # (retro×N, then fireacademy×N, …), so the total is iterations * n. + "environment_order": "round_robin", # round_robin | grouped "on_step_failure": "abort_iteration", # continue | abort_iteration | abort_mission "ready": {"timeout_s": 600, "poll_interval_s": 5}, # scope "gcs": one recorder on GCS domain 0 → one mcap with every robot's # bridged topics (default). scope "robot": one recorder + one mcap per - # robot on its own domain (for unbridged/high-rate topics). + # robot on its own domain (for unbridged/high-rate topics). scope "both": + # the GCS recorder AND the per-robot recorders together. "record": {"enabled": True, "scope": "gcs"}, # How the stack is brought up. Either (or both): # services: [isaac-sim, robot-desktop, gcs] → ./airstack.sh up @@ -118,6 +137,22 @@ } +class MissionStop(Exception): + """Raised in the main thread on SIGINT/SIGTERM (e.g. `airstack + osmo:stop`): aborts the current step, finalizes recordings, collects + artifacts, and brings the stack down.""" + + +STOP_EVENT = threading.Event() + + +def _stop_handler(signum, frame): + if STOP_EVENT.is_set(): + return # already stopping — don't interrupt artifact collection + STOP_EVENT.set() + raise MissionStop(f"stop requested (signal {signum})") + + def log(msg): print(f"[mission {datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True) @@ -210,11 +245,30 @@ def load_mission(path): if merged["on_step_failure"] not in ("continue", "abort_iteration", "abort_mission"): raise ValueError(f"on_step_failure must be continue|abort_iteration|abort_mission, " f"got {merged['on_step_failure']!r}") + if merged["environment_order"] not in ("round_robin", "grouped"): + raise ValueError(f"environment_order must be round_robin|grouped, " + f"got {merged['environment_order']!r}") + try: + merged["iteration_attempts"] = int(merged["iteration_attempts"]) + except (TypeError, ValueError): + raise ValueError(f"iteration_attempts must be a positive integer, " + f"got {merged['iteration_attempts']!r}") + if merged["iteration_attempts"] < 1: + raise ValueError(f"iteration_attempts must be >= 1, " + f"got {merged['iteration_attempts']}") if merged["command_route"] not in ("gcs", "robot"): raise ValueError(f"command_route must be gcs|robot, got {merged['command_route']!r}") - if merged["record"].get("scope", "gcs") not in ("gcs", "robot"): - raise ValueError(f"record.scope must be gcs|robot, " + if merged["record"].get("scope", "gcs") not in ("gcs", "robot", "both"): + raise ValueError(f"record.scope must be gcs|robot|both, " f"got {merged['record'].get('scope')!r}") + envs = merged.get("environments") or [] + if not isinstance(envs, list): + raise ValueError("environments must be a list of environment entries") + for e in envs: + if not isinstance(e, dict) or "name" not in e: + raise ValueError(f"each environments entry must be a mapping with a " + f"'name' key, got {e!r}") + merged["environments"] = envs for step in merged["steps"]: action = step.get("action") if isinstance(step, dict) else None if action: @@ -226,6 +280,10 @@ def load_mission(path): f"task '{action.get('task')}' is not bridged by the GCS action_relay " f"({', '.join(sorted(GCS_RELAY_TASKS))}); add `via: robot` to send it " f"directly on the robot's domain") + if action.get("pass_on_feedback") and via != "gcs": + raise ValueError( + "pass_on_feedback requires via: gcs (feedback liveness is only " + "tracked on the GCS action_relay path)") return merged @@ -261,6 +319,96 @@ def uses_gcs_route(mission): return False +# ── per-iteration environment cycling ────────────────────────────────────── +# An optional top-level `environments:` list lets one mission run several +# scenes. environment_order=round_robin: iteration i uses environments[(i-1) % +# len] (`iterations` = total). environment_order=grouped: each env runs +# `iterations` times in a row (total = iterations * len). Within an entry, +# UPPERCASE keys (ENV_URL, SPAWN_POLY, …) are exported as env vars before +# `airstack up` (so they reach the isaac-sim container); any key can be +# referenced from a step via a {{env.KEY}} placeholder (e.g. the search_area +# polygon, which must track the scene). + +_ENV_PLACEHOLDER = re.compile(r"\{\{\s*env\.([A-Za-z0-9_]+)\s*\}\}") + + +def env_exports(env_entry): + """UPPERCASE keys of an environment entry → env vars for `airstack up`. + Structured values are JSON-encoded so SPAWN_POLY can be written either as a + YAML list or as a JSON string in the mission spec.""" + return {k: (v if isinstance(v, str) else json.dumps(v)) + for k, v in env_entry.items() if k.isupper()} + + +def substitute_env(obj, env_entry): + """Replace {{env.KEY}} placeholders in a (copy of a) step spec with values + from the current environment entry. A string that is *exactly* one + placeholder yields the raw value (so {{env.search_area}} can resolve to a + dict); a placeholder embedded in a larger string is substituted textually.""" + if isinstance(obj, str): + whole = _ENV_PLACEHOLDER.fullmatch(obj.strip()) + if whole: + return _lookup_env(env_entry, whole.group(1)) + + def repl(m): + val = _lookup_env(env_entry, m.group(1)) + return val if isinstance(val, str) else json.dumps(val) + return _ENV_PLACEHOLDER.sub(repl, obj) + if isinstance(obj, dict): + return {k: substitute_env(v, env_entry) for k, v in obj.items()} + if isinstance(obj, list): + return [substitute_env(v, env_entry) for v in obj] + return obj + + +def _lookup_env(env_entry, key): + if key not in env_entry: + raise ValueError(f"{{{{env.{key}}}}} referenced in a step but not found in " + f"environment '{env_entry.get('name')}' " + f"(available keys: {sorted(env_entry)})") + return env_entry[key] + + +# ── iteration naming (env + method in the dir name) ───────────────────────── +# `method:` is an optional free-form label (on an environment entry or at the +# mission top level) for missions that compare algorithms/configurations across +# iterations — it lands in the iteration dir name and summary.json so runs stay +# distinguishable after fetch. + +def _slug(s): + """Filesystem-safe lowercase token.""" + s = re.sub(r"[^A-Za-z0-9._-]+", "-", str(s).strip().lower()) + return s.strip("-") + + +def resolve_method(mission, env_entry): + """Normalized method label. Explicit `method:` on the environment entry + wins, then a mission-level `method:`; 'na' (omitted from names) otherwise.""" + if env_entry and env_entry.get("method"): + return _slug(env_entry["method"]) + if mission.get("method"): + return _slug(mission["method"]) + return "na" + + +def iter_dir_name(i, env_entry, method): + """iter_NNN[__][__]. The iter_NNN prefix is kept so `iter_*` + globs and numeric ordering still work; a trailing method suffix already in + the env name is dropped to avoid e.g. scene_a_frontier__frontier.""" + parts = [f"iter_{i:03d}"] + env_slug = _slug(env_entry["name"]) if env_entry and env_entry.get("name") else "" + if env_slug: + for sep in ("_", "-"): + if method != "na" and env_slug.endswith(f"{sep}{method}"): + env_slug = env_slug[: -len(method) - 1] + break + if env_slug: + parts.append(env_slug) + if method and method != "na": + parts.append(method) + return "__".join(parts) + + # ── stack lifecycle ──────────────────────────────────────────────────────── class Stack: @@ -283,6 +431,12 @@ def __init__(self, airstack_root, mission): self.num_robots = int(self.env["NUM_ROBOTS"]) self.setup_bash = mission["robot_setup_bash"] + def apply_env(self, overrides): + """Merge per-iteration env-var overrides (from an `environments:` entry) + into the bring-up environment so the next `airstack up` sees them.""" + for k, v in overrides.items(): + self.env[k] = v if isinstance(v, str) else str(v) + def _airstack(self, verb, timeout, extra_args=()): log(f"airstack {verb} {' '.join(extra_args)} " f"(NUM_ROBOTS={self.env['NUM_ROBOTS']}, " @@ -417,6 +571,10 @@ class Recorder: scope "robot": one recorder per robot inside robot container 1, each on that robot's DDS domain — for topics that aren't bridged to the GCS. + scope "both": the GCS recorder and the per-robot recorders together — + one mcap with the fleet view from domain 0 plus one raw-data mcap per + robot domain. + Each recorder is started detached with its PID dropped to a file, and stopped with SIGTERM so rosbag2 finalizes the mcap cleanly. (Not SIGINT: jobs backgrounded from a non-interactive shell have SIGINT set to @@ -432,13 +590,19 @@ def __init__(self, robot_container, mission, num_robots, setup_bash): # (container, domain_id, tag) per active recorder. self.active = [] - def _topic_selection(self, robots): + def _topic_selection(self, robots, scope): """Build the `ros2 bag record` topic args; `robots` is the list of robot indices whose {robot}/{n} placeholders to expand (unioned, - order-preserving, deduplicated).""" + order-preserving, deduplicated). `scope` picks the default topic set + for the recorder being started ("gcs" or "robot").""" if self.cfg.get("all"): - return "-a" - default = (DEFAULT_GCS_RECORD_TOPICS if self.scope == "gcs" + sel = "-a" + # record.exclude: regex of topics to drop from `-a`. + exclude = self.cfg.get("exclude") + if exclude: + sel += f" --exclude-regex {shlex.quote(exclude)}" + return sel + default = (DEFAULT_GCS_RECORD_TOPICS if scope == "gcs" else DEFAULT_ROBOT_RECORD_TOPICS) topics = [] for t in self.cfg.get("topics", default): @@ -476,30 +640,47 @@ def _start_one(self, container, domain_id, tag, selection, setup_bash): r = docker_exec(container, inner, timeout=30) if "RECORDER_ALIVE" in r.stdout: self.active.append((container, tag)) - n_topics = "all topics" if selection == "-a" else f"{len(selection.split())} topics" + n_topics = ("all topics" if selection.startswith("-a") + else f"{len(selection.split())} topics") log(f"recording [{tag}] in {container} (domain {domain_id}) " f"→ {out_dir} ({n_topics})") - else: - log(f"WARN: recorder [{tag}] failed to start / exited immediately: " - f"{tail(r.stdout + r.stderr, 6)}") + return True + log(f"WARN: recorder [{tag}] failed to start / exited immediately: " + f"{tail(r.stdout + r.stderr, 6)}") + return False def start(self): if not self.cfg.get("enabled", True): log("recording disabled by mission spec") return robots = list(range(1, self.num_robots + 1)) - if self.scope == "gcs": + failed = [] + if self.scope in ("gcs", "both"): gcs = gcs_container() - if not gcs: - log("WARN: record.scope is 'gcs' but no gcs container is running — " - "recording skipped (bring up the gcs service or use scope: robot)") - return - self._start_one(gcs, 0, "gcs", self._topic_selection(robots), - GCS_SETUP_BASH) - else: + if gcs: + if not self._start_one(gcs, 0, "gcs", + self._topic_selection(robots, "gcs"), + GCS_SETUP_BASH): + failed.append("gcs") + else: + log("WARN: record.scope includes 'gcs' but no gcs container is " + "running — GCS recording skipped (bring up the gcs service " + "or use scope: robot)") + failed.append("gcs (no container)") + if self.scope == "gcs" and not self.cfg.get("required"): + return + if self.scope in ("robot", "both"): for n in robots: - self._start_one(self.robot_container, n, f"robot_{n}", - self._topic_selection([n]), self.setup_bash) + if not self._start_one(self.robot_container, n, f"robot_{n}", + self._topic_selection([n], "robot"), + self.setup_bash): + failed.append(f"robot_{n}") + # record.required: a mission whose deliverable is the bags shouldn't + # silently fly unrecorded. + if failed and self.cfg.get("required"): + raise RuntimeError( + f"record.required is set and recorder(s) failed to start: " + f"{failed} — aborting iteration") def stop(self): for container, tag in self.active: @@ -570,6 +751,25 @@ def _run_one(stack, robot_container, target, cmd, timeout, expect_success): "output_tail": tail(r.stdout + r.stderr)} +def _run_one_retry(stack, robot_container, target, cmd, timeout, expect_success, + attempts, retry_delay_s): + """_run_one with up to `attempts` tries (retry_delay_s between failures).""" + res = {"target": target, "exit": 1, "ok": False, + "output_tail": "not attempted — mission stop requested", "attempts": 0} + for attempt in range(1, max(1, attempts) + 1): + if STOP_EVENT.is_set(): + break + res = _run_one(stack, robot_container, target, cmd, timeout, expect_success) + res["attempts"] = attempt + if res["ok"]: + break + if attempt < attempts and not STOP_EVENT.is_set(): + log(f"run [{target}] failed (attempt {attempt}/{attempts}); " + f"retrying in {retry_delay_s}s") + time.sleep(retry_delay_s) + return res + + def run_step(stack, container, step_spec, step_index): """Execute one step; returns a result dict with ok: bool.""" record = {"index": step_index, "spec": step_spec, @@ -594,6 +794,27 @@ def run_step(stack, container, step_spec, step_index): goal_obj = yaml.safe_load(goal) if isinstance(goal, str) else goal goal_json = json.dumps(goal_obj or {}) timeout = float(spec.get("timeout_s", 120)) + # Per-robot retries for transient rejections (no GPS fix yet, PX4 + # position not converged, goal lost on the relay's volatile sub). + max_attempts = int(spec.get("attempts", 3)) + retry_delay_s = float(spec.get("retry_delay_s", 10)) + # via: gcs only — no relay_feedback AND no result within this window + # after publishing ⇒ the goal is presumed lost; fail fast and retry. + feedback_timeout_s = int(spec.get("feedback_timeout_s", 15)) + # cancel_on_timeout (via:gcs only): on `timeout_s`, publish a CancelGoal + # to the relay's .../cancel topic and wait up to cancel_grace_s for the + # cancelled result, so an on-cancel finalize (e.g. semantic_search's + # metrics) can run and return. + cancel_on_timeout = bool(spec.get("cancel_on_timeout", False)) + cancel_grace_s = int(spec.get("cancel_grace_s", 120)) + # pass_on_feedback (via:gcs only): treat the step as passed as soon as the + # action is confirmed to have actually started running — i.e. any + # relay_feedback was seen (or a result came back), regardless of whether + # the result is success or failure. The ONLY failure left is "goal lost / + # never ran" (no feedback and no result within the retries), which is what + # then fails the step (→ iteration redo when iteration_attempts>1). Use + # this when you only care that the task ran, not its outcome. + pass_on_feedback = bool(spec.get("pass_on_feedback", False)) robots = step_robots(spec, stack.num_robots) log(f"step {step_index}: action {task} {goal_json} via {via} → robots {robots}") @@ -607,32 +828,102 @@ def run_step(stack, container, step_spec, step_index): log(f"step {step_index}: FAILED (no gcs container)") return record + # The result echo must outlive `timeout` to catch the post-cancel + # result. + echo_timeout = int(timeout + cancel_grace_s) if cancel_on_timeout \ + else int(timeout) + def send(n): - # Same path as Foxglove: publish String JSON on - # //tasks//goal (GCS domain 0); the per-robot - # action_relay forwards it as a typed action goal on domain N - # and reports {"success": ..., "message": ...} on - # .../relay_result. Subscribe to the result *before* - # publishing the goal so a fast result can't be missed. + # Foxglove's path: String JSON on //tasks//goal, + # result on .../relay_result. relay_result is latched, so a + # fresh subscriber gets the PREVIOUS goal's result — count + # messages before publishing and only accept a NEW one. + # relay_feedback is the liveness signal: nothing there (and + # no result) within feedback_timeout_s ⇒ goal lost, bail out + # so the retry loop re-sends. base = f"/robot_{n}/tasks/{task}" result_file = f"/tmp/relay_result_{task}_{n}.out" + fb_file = f"/tmp/relay_feedback_{task}_{n}.out" msg_yaml = json.dumps({"data": expand(goal_json, n)}) + cancel_yaml = json.dumps({"data": "osmo: timeout cancel"}) script = ( - f"rm -f {result_file}\n" - f"( timeout {int(timeout)} ros2 topic echo --once --field data " - f"{base}/relay_result > {result_file} 2>&1 ) &\n" + f"rm -f {result_file} {fb_file}\n" + f"touch {result_file} {fb_file}\n" + f"( timeout {echo_timeout} ros2 topic echo --field data " + f"{base}/relay_result >> {result_file} 2>&1 ) &\n" f"sub=$!\n" + f"( timeout {echo_timeout} ros2 topic echo --field data " + f"{base}/relay_feedback >> {fb_file} 2>&1 ) &\n" + f"fb_sub=$!\n" f"sleep 3\n" - f"ros2 topic pub --once {base}/goal std_msgs/msg/String " + f"pre=$(grep -c '^---' {result_file})\n" + # -w 1: wait for the relay's (VOLATILE) goal sub to match + # before publishing; --keep-alive 3: outlive the publish so + # the RELIABLE handshake flushes (0.1s default drops goals). + f"ros2 topic pub --once -w 1 --keep-alive 3 " + f"{base}/goal std_msgs/msg/String " f"{shlex.quote(msg_yaml)} > /dev/null\n" - f"wait $sub\n" - f"cat {result_file}" + f"sent=$(date +%s)\n" + f"fb_seen=0\n" + f"cancelled=0\n" + f"deadline=$(( sent + {int(timeout)} ))\n" + f"while :; do\n" + f" cur=$(grep -c '^---' {result_file})\n" + f" if [ \"$cur\" -gt \"$pre\" ]; then break; fi\n" + f" now=$(date +%s)\n" + f" if [ \"$fb_seen\" -eq 0 ] && " + f"[ \"$(grep -c '^---' {fb_file})\" -gt 0 ]; then fb_seen=1; fi\n" + f" if [ \"$cancelled\" -eq 0 ] && [ \"$fb_seen\" -eq 0 ] && " + f"[ $(( now - sent )) -ge {feedback_timeout_s} ]; then\n" + f" kill $sub $fb_sub 2>/dev/null\n" + f" echo 'no relay_feedback within {feedback_timeout_s}s " + f"of goal publish — goal presumed lost'\n" + f" exit 0\n" + f" fi\n" + f" if [ $now -ge $deadline ]; then\n" + # Deadline: cancel + keep waiting if cancel_on_timeout, else stop. + f" if [ {1 if cancel_on_timeout else 0} -eq 1 ] && " + f"[ \"$cancelled\" -eq 0 ]; then\n" + f" echo 'osmo: {int(timeout)}s timeout — sending cancel'\n" + f" ros2 topic pub --once -w 1 --keep-alive 3 " + f"{base}/cancel std_msgs/msg/String " + f"{shlex.quote(cancel_yaml)} > /dev/null\n" + f" cancelled=1\n" + f" deadline=$(( now + {cancel_grace_s} ))\n" + f" else\n" + f" break\n" + f" fi\n" + f" fi\n" + f" kill -0 $sub 2>/dev/null || break\n" + f" sleep 1\n" + f"done\n" + f"kill $sub $fb_sub 2>/dev/null\n" + # Surface whether the action ever produced feedback (it ran), + # so the runner can pass on liveness when pass_on_feedback is + # set. The early no-feedback bail above exits before here, so + # this only prints when feedback was genuinely observed. + f"if [ \"$fb_seen\" -eq 1 ]; then echo OSMO_FEEDBACK_SEEN; fi\n" + f"cur=$(grep -c '^---' {result_file})\n" + f"if [ \"$cur\" -gt \"$pre\" ]; then " + f"grep -v '^---' {result_file} | tail -n 1; " + f"else echo 'no relay_result within {echo_timeout}s'; fi" ) r = ros2_exec(gcs, script, domain_id=0, setup_bash=GCS_SETUP_BASH, - timeout=int(timeout + 30)) - ok = '"success": true' in r.stdout - return n, {"exit": r.returncode, "ok": ok, - "output_tail": tail(r.stdout + r.stderr)} + timeout=int(echo_timeout + 30)) + # For cancel_on_timeout, a returned (success=false) cancel result + # is the intended outcome — treat it as ok so the step doesn't + # retry the search or trip on_step_failure. + got_result = '"success"' in r.stdout + feedback_seen = 'OSMO_FEEDBACK_SEEN' in r.stdout + ok = ('"success": true' in r.stdout + or (cancel_on_timeout and got_result)) + # pass_on_feedback: ran-at-all (feedback seen, or any result + # came back) is enough — outcome doesn't matter. + if pass_on_feedback: + ok = ok or feedback_seen or got_result + return {"exit": r.returncode, "ok": ok, + "feedback_seen": feedback_seen, + "output_tail": tail(r.stdout + r.stderr)} else: action_type = spec.get("type", task_action_type(task)) @@ -641,11 +932,38 @@ def send(n): f"{action_type} {shlex.quote(expand(goal_json, n))}") r = ros2_exec(container, cmd, domain_id=n, setup_bash=stack.setup_bash, timeout=int(timeout + 15)) - return n, {"exit": r.returncode, "ok": action_ok(r.stdout), - "output_tail": tail(r.stdout + r.stderr)} - - with ThreadPoolExecutor(max_workers=len(robots)) as pool: - results = dict(pool.map(send, robots)) + return {"exit": r.returncode, "ok": action_ok(r.stdout), + "output_tail": tail(r.stdout + r.stderr)} + + def send_with_retry(n): + res = {"ok": False, "exit": -1, + "output_tail": "not attempted — mission stop requested"} + attempt = 0 + for attempt in range(1, max_attempts + 1): + if STOP_EVENT.is_set(): + break + log(f"step {step_index}: sending {task} to robot_{n} " + f"(attempt {attempt}/{max_attempts})") + res = send(n) + if res["ok"]: + log(f"step {step_index}: {task} robot_{n} succeeded" + + (f" on attempt {attempt}" if attempt > 1 else "")) + break + log(f"step {step_index}: {task} robot_{n} attempt {attempt} " + f"FAILED: {tail(res.get('output_tail', ''), 1)}") + if attempt < max_attempts and not STOP_EVENT.is_set(): + time.sleep(retry_delay_s) + res["attempts"] = attempt + return n, res + + # No context manager: on MissionStop the `with` form would block in + # shutdown(wait=True) until the in-flight docker execs hit their + # timeouts. The workers die when `airstack down` kills the containers. + pool = ThreadPoolExecutor(max_workers=len(robots)) + try: + results = dict(pool.map(send_with_retry, robots)) + finally: + pool.shutdown(wait=False, cancel_futures=True) record.update(type="action", task=task, via=via, per_robot=results, ok=all(v["ok"] for v in results.values())) @@ -654,6 +972,8 @@ def send(n): cmd = spec["cmd"] timeout = float(spec.get("timeout_s", 60)) expect = spec.get("expect_success", True) + attempts = int(spec.get("attempts", 1)) + retry_delay_s = float(spec.get("retry_delay_s", 10)) # `container` may reference {n}/{robot} to fan out over robots. If it's # omitted, default to robot_{n} when the command is per-robot (has a # placeholder) and robot_1 otherwise. A step fans out over `robots` @@ -664,13 +984,15 @@ def send(n): log(f"step {step_index}: run [{target}] {cmd} → robots {robots}") results = {} for n in robots: - results[n] = _run_one(stack, container, expand(target, n), - expand(cmd, n), timeout, expect) + results[n] = _run_one_retry(stack, container, expand(target, n), + expand(cmd, n), timeout, expect, + attempts, retry_delay_s) record.update(type="run", per_robot=results, ok=all(v["ok"] for v in results.values())) else: log(f"step {step_index}: run [{target}] {cmd}") - res = _run_one(stack, container, target, cmd, timeout, expect) + res = _run_one_retry(stack, container, target, cmd, timeout, expect, + attempts, retry_delay_s) record.update(type="run", **res) elif "topic_pub" in step_spec: @@ -719,9 +1041,83 @@ def send(n): # ── artifacts ────────────────────────────────────────────────────────────── +# Navigation / control-stack node-name globs. These log via rcl to ~/.ros/log +# (not /tmp), so they're collected separately — they're what reveals why a +# planner's plan isn't becoming motion (e.g. a robot stuck mid-mission). Task +# nodes are here too so their rcl log is captured as a SECOND path independent +# of the /tmp tee (TEE_LOG_GLOBS) — if one collection path misses, the other +# still shows why the drone wasn't commanded. +CONTROL_STACK_GLOBS = ("droan_gl*", "droan_local_planner*", "trajectory_controller*", + "fixed_trajectory*", "takeoff_landing*", "mavros*", + "semantic_search_task*") + + +# /tmp tee logs written by task-spawned subprocesses and the GCS +# relay/gossip/ddsrouter — the stdout that rcl logging doesn't capture. +TEE_LOG_GLOBS = ("/tmp/gossip_*.log", "/tmp/ddsrouter_*.log", "/tmp/relay_*.log") + + +def _copy_tmp_tee_logs(dest_dir): + """Copy the raw /tmp tee logs from each container into //. + Idempotent (docker cp overwrites), so it is safe to call repeatedly — the + snapshot loop calls it periodically so a hard pod cancel mid-iteration still + leaves the most recent stdout tees on disk.""" + targets = list(robot_containers()) + gcs = gcs_container() + if gcs: + targets.append(gcs) + total = 0 + for name in targets: + r = docker_exec(name, "ls " + " ".join(TEE_LOG_GLOBS) + " 2>/dev/null", + timeout=15) + files = [f for f in r.stdout.split() if f] + if not files: + continue + out = dest_dir / name + out.mkdir(parents=True, exist_ok=True) + for f in files: + sh(["docker", "cp", f"{name}:{f}", str(out)], timeout=120) + total += len(files) + return total + + +def snapshot_task_logs(dest_dir): + """Copy the raw gossip/ddsrouter/relay tees (written to /tmp inside each + container) into //, plus the navigation / + control-stack node logs that rcl writes under ~/.ros/log rather than /tmp. + Includes the GCS container so the GCS-side gossip ddsrouter log is captured.""" + targets = list(robot_containers()) + gcs = gcs_container() + if gcs: + targets.append(gcs) + _copy_tmp_tee_logs(dest_dir) + name_pred = " -o ".join(f"-name '{g}'" for g in CONTROL_STACK_GLOBS) + tar = "/tmp/control_stack_logs.tar.gz" + for name in targets: + r = docker_exec(name, "ls " + " ".join(TEE_LOG_GLOBS) + " 2>/dev/null", + timeout=15) + files = [f for f in r.stdout.split() if f] + out = dest_dir / name + made = docker_exec( + name, + f"f=$(find /root/.ros/log -type f \\( {name_pred} \\) 2>/dev/null); " + f"[ -n \"$f\" ] && tar czf {tar} $f 2>/dev/null && echo COLLECTED", + timeout=30) + if "COLLECTED" in (made.stdout or ""): + out.mkdir(parents=True, exist_ok=True) + sh(["docker", "cp", f"{name}:{tar}", + str(out / "control_stack_logs.tar.gz")], timeout=120) + files = files + ["control_stack_logs.tar.gz"] + if files: + log(f"collected {len(files)} node log(s) from {name}") + + def snapshot_container_logs(dest_dir): dest_dir.mkdir(parents=True, exist_ok=True) - for name in list_containers(name_pattern="airstack", all_states=True): + pats = ("airstack", "isaac-sim", "ms-airsim") + names = [n for n in list_containers(all_states=True) + if any(p in n for p in pats)] + for name in names: r = sh(["docker", "logs", name], timeout=120) (dest_dir / f"{name}.log").write_text( (r.stdout or "") + (("\n--- stderr ---\n" + r.stderr) if r.stderr else ""), @@ -757,12 +1153,141 @@ def resolve_results_root(airstack_root): # ── main loop ────────────────────────────────────────────────────────────── +# Per-container health snapshot. Sections are emitted on tagged lines so the +# multi-line ps output stays parseable. cgroup paths are v2; absent on v1 (the +# fields just come back empty). +_SNAP_CMD = r""" +echo "T|$(date '+%F %T')" +echo "DDS|procs=$(pgrep -c ddsrouter 2>/dev/null) estab=$(ss -Htn 2>/dev/null | grep -c ESTAB) ddstcp=$(ss -Htnp 2>/dev/null | grep -c ddsrouter) cpu=$(ps -C ddsrouter -o %cpu= 2>/dev/null | tr '\n' ',')" +echo "MEM|cur=$(cat /sys/fs/cgroup/memory.current 2>/dev/null) max=$(cat /sys/fs/cgroup/memory.max 2>/dev/null) oom_kill=$(awk '/^oom_kill /{print $2}' /sys/fs/cgroup/memory.events 2>/dev/null)" +echo "PS|" +ps -eo rss=,args= --sort=-rss 2>/dev/null | head -12 +""" + +# Processes whose early death we want to catch. Matched as substrings of the +# full command line (python nodes all run as `python3 ...`). Extend per-mission +# with any long-running node whose silent crash would otherwise only show up as +# a stuck drone. +_WATCH_PROCS = ("ddsrouter", "gossip") + + +def _parse_snapshot(out): + ts = dds = mem = "" + oom = None + ps_rows = [] + in_ps = False + for ln in out.splitlines(): + if ln.startswith("T|"): + ts = ln[2:].strip() + elif ln.startswith("DDS|"): + dds = ln[4:].strip() + elif ln.startswith("MEM|"): + mem = ln[4:].strip() + m = re.search(r"oom_kill=(\d+)", mem) + oom = int(m.group(1)) if m else None + elif ln.startswith("PS|"): + in_ps = True + elif in_ps: + p = ln.split(None, 1) + if len(p) == 2 and p[0].isdigit(): + ps_rows.append((int(p[0]), p[1].strip())) + return ts, dds, mem, oom, ps_rows + + +def _format_resources(mem, ps_rows): + def mb(v): + return f"{int(v) // 1048576}MB" if v.isdigit() else (v or "?") + m = re.search(r"cur=(\S+) max=(\S+)", mem) + memstr = f"mem={mb(m.group(1))}/{mb(m.group(2))}" if m else mem + oom = re.search(r"oom_kill=\S+", mem) + key = {k: 0 for k in _WATCH_PROCS} + for rss, args in ps_rows: + for k in _WATCH_PROCS: + if k in args: + key[k] += rss + keystr = " ".join(f"{k}={v // 1024}MB" for k, v in key.items()) + top = [] + for rss, args in ps_rows[:5]: + label = next((k for k in _WATCH_PROCS if k in args), + args.split()[0].rsplit("/", 1)[-1][:16]) + top.append(f"{label}:{rss // 1024}MB") + return f"{memstr} {oom.group(0) if oom else ''} | {keystr} | top: {' '.join(top)}", key + + +def _capture_oom_dmesg(iter_dir, container, ts, why): + """Append the kernel OOM-killer trail (host ring buffer, readable from the + privileged container) when a watched process dies or oom_kill increments.""" + r = docker_exec( + container, + "dmesg -T 2>/dev/null | grep -iE 'killed process|out of memory|oom-kill|" + "invoked oom|memory cgroup' | tail -40", timeout=15) + txt = (r.stdout or "").strip() + block = (f"### {container} @ {ts} — {why}\n" + f"{txt or '(dmesg unavailable in container — run host: dmesg -T | grep -i oom)'}\n") + with open(iter_dir / "oom_dmesg.log", "a", encoding="utf-8") as f: + f.write(block) + + +def transport_snapshot_loop(iter_dir, stop_event, interval_s=5): + """Periodic per-container health snapshot for debugging early process death: + - transport.log : ddsrouter procs + domain-99 TCP (gossip dropout) + - resources.log : cgroup memory, oom_kill counter, watched + heaviest procs + - oom_dmesg.log : kernel OOM trail, captured when a watched process + vanishes or the cgroup oom_kill counter increments + """ + iter_dir.mkdir(parents=True, exist_ok=True) + tlog, rlog = iter_dir / "transport.log", iter_dir / "resources.log" + prev_oom, prev_key = {}, {} + # Periodically copy the /tmp tee logs into iter_dir/logs so a hard pod cancel + # (SIGKILL — run_iteration's finally never runs) still leaves recent + # stdout tees. ~every 30s regardless of the health interval. + tee_every = max(1, round(30 / interval_s)) + tick = 0 + while not stop_event.is_set(): + targets = list(robot_containers()) + gcs = gcs_container() + if gcs: + targets.append(gcs) + tlines, rlines = [], [] + for name in targets: + r = docker_exec(name, _SNAP_CMD, timeout=15) + ts, dds, mem, oom, ps_rows = _parse_snapshot(r.stdout or "") + if not ts: + continue + tlines.append(f"{name} [{ts}] {dds}") + res, key = _format_resources(mem, ps_rows) + rlines.append(f"{name} [{ts}] {res}") + if oom is not None: + if prev_oom.get(name, oom) < oom: + _capture_oom_dmesg(iter_dir, name, ts, f"cgroup oom_kill -> {oom}") + prev_oom[name] = oom + if ps_rows: # only trust presence when we got a real ps sample + for k in _WATCH_PROCS: + if prev_key.get((name, k), 0) > 0 and key[k] == 0: + _capture_oom_dmesg(iter_dir, name, ts, f"{k} vanished") + prev_key[(name, k)] = key[k] + if tlines: + with open(tlog, "a", encoding="utf-8") as f: + f.write("\n".join(tlines) + "\n") + if rlines: + with open(rlog, "a", encoding="utf-8") as f: + f.write("\n".join(rlines) + "\n") + if tick % tee_every == 0: + try: + _copy_tmp_tee_logs(iter_dir / "logs") + except Exception as e: + log(f"WARN: periodic tee-log copy failed: {e}") + tick += 1 + stop_event.wait(interval_s) + + def run_iteration(stack, mission, iter_dir): """One full up → ready → record → steps → collect → down cycle. Returns the iteration summary dict; never raises (failures are data).""" summary = {"status": "passed", "steps_ok": 0, "steps_failed": 0} recorder = None container = None + snap_stop = None t0 = time.time() try: # `airstack status`; if a stack is already up (baked entrypoint on a @@ -774,6 +1299,10 @@ def run_iteration(stack, mission, iter_dir): container = containers[0] summary["up_duration_s"] = round(time.time() - t0, 2) + snap_stop = threading.Event() + threading.Thread(target=transport_snapshot_loop, + args=(iter_dir, snap_stop), daemon=True).start() + ready_at = stack.wait_ready(container) write_json(iter_dir / "ready.json", ready_at) @@ -788,6 +1317,13 @@ def run_iteration(stack, mission, iter_dir): summary["steps_ok"] += 1 continue summary["steps_failed"] += 1 + # An `optional: true` step records its failure but never fails the + # iteration (so it doesn't trip on_step_failure or an iteration redo) + # — e.g. a post-search land after the gating step already ran. + if isinstance(step_spec, dict) and step_spec.get("optional"): + log(f"step {i}: failed but marked optional — continuing") + summary.setdefault("optional_failures", []).append(i) + continue summary["status"] = "failed" policy = mission["on_step_failure"] if policy == "continue": @@ -797,12 +1333,19 @@ def run_iteration(stack, mission, iter_dir): break write_json(iter_dir / "steps.json", steps) + except MissionStop as e: + summary["status"] = "stopped" + summary["abort_mission"] = True + log(f"STOP: {e} — finalizing recordings and collecting artifacts") + except Exception as e: summary["status"] = "error" summary["error"] = str(e) log(f"ERROR: iteration aborted: {e}") finally: + if snap_stop is not None: + snap_stop.set() # Artifact collection happens even on failure — a failed flight's # bag is usually the most interesting one. if recorder is not None: @@ -810,6 +1353,7 @@ def run_iteration(stack, mission, iter_dir): recorder.collect(iter_dir / "bags") if container is not None or list_containers("airstack", all_states=True): snapshot_container_logs(iter_dir / "logs") + snapshot_task_logs(iter_dir / "logs") stack.down() summary["duration_s"] = round(time.time() - t0, 2) write_json(iter_dir / "iteration.json", summary) @@ -828,8 +1372,16 @@ def main(): mission = load_mission(args.mission_file) stack = Stack(args.airstack_root, mission) + # Graceful stop: SIGINT/SIGTERM (Ctrl-C locally, `airstack osmo:stop` on + # a pod) aborts the current step but still finalizes the mcaps, collects + # bags + logs, and brings the stack down. + signal.signal(signal.SIGINT, _stop_handler) + signal.signal(signal.SIGTERM, _stop_handler) + log(f"mission '{mission['name']}': {mission['iterations']} iteration(s), " - f"{len(mission['steps'])} step(s), {stack.num_robots} robot(s)") + f"{len(mission['steps'])} step(s), {stack.num_robots} robot(s)" + + (f", up to {mission['iteration_attempts']} attempt(s)/iteration" + if mission["iteration_attempts"] > 1 else "")) if args.dry_run: print(yaml.safe_dump(mission, sort_keys=False)) return 0 @@ -839,18 +1391,81 @@ def main(): run_dir.mkdir(parents=True) log(f"results → {run_dir}") + environments = mission.get("environments") or [] + n_iter = mission["iterations"] + # Build the per-iteration environment schedule. round_robin: cycle envs, + # `iterations` is the total. grouped: run each env `iterations` times in a + # row, so the total is iterations * len(environments). + if not environments: + schedule = [None] * n_iter + elif mission["environment_order"] == "grouped": + schedule = [env for env in environments for _ in range(n_iter)] + else: + schedule = [environments[i % len(environments)] for i in range(n_iter)] + total = len(schedule) + if environments: + log(f"{mission['environment_order']} over {len(environments)} " + f"environment(s) → {total} iteration(s): " + f"{[e['name'] for e in schedule]}") + iterations = [] - for i in range(1, mission["iterations"] + 1): - log(f"━━━ iteration {i}/{mission['iterations']} ━━━") - iter_dir = run_dir / f"iter_{i:03d}" - summary = run_iteration(stack, mission, iter_dir) + max_attempts = mission["iteration_attempts"] + for i, env_entry in enumerate(schedule, start=1): + log(f"━━━ iteration {i}/{total} ━━━") + method = resolve_method(mission, env_entry) + iter_dir = run_dir / iter_dir_name(i, env_entry, method) + # Apply this iteration's environment (scene URL + spawn area as env vars + # for `airstack up`; search_area et al. templated into steps). + if env_entry is not None: + log(f"environment: {env_entry['name']} " + f"(ENV_URL={env_entry.get('ENV_URL')})") + stack.apply_env(env_exports(env_entry)) + iter_mission = {**mission, "steps": substitute_env(mission["steps"], env_entry)} + else: + iter_mission = mission + + # Iteration redo: re-run the SAME iteration up to iteration_attempts + # times until it passes. A passed run, a manual stop, or an explicit + # abort_mission is never retried; a failed/errored run is (the stack is + # already down by run_iteration's finally, so each attempt is a clean + # down→up→fly→down cycle — and with a fixed SPAWN_SEED the spawn layout + # is reproduced exactly). + attempt = 0 + while True: + attempt += 1 + if attempt > 1: + log(f"iteration {i}: attempt {attempt}/{max_attempts}") + summary = run_iteration(stack, iter_mission, iter_dir) + done = (summary["status"] in ("passed", "stopped") + or summary.get("abort_mission") + or attempt >= max_attempts) + if done: + break + # Failed with retries left: stash this attempt's artifacts so the + # canonical iter_NNN ends up holding the final attempt. + failed_dir = run_dir / f"{iter_dir.name}_failed_attempt_{attempt}" + try: + if iter_dir.exists(): + iter_dir.rename(failed_dir) + log(f"iteration {i} {summary['status']} on attempt {attempt}" + f"/{max_attempts}; artifacts → {failed_dir.name}; redoing") + except OSError as e: + log(f"WARN: could not stash failed attempt dir ({e}); " + f"redoing into {iter_dir.name}") + summary["iteration"] = i + summary["attempts"] = attempt + summary["method"] = method + if env_entry is not None: + summary["environment"] = env_entry["name"] iterations.append(summary) write_json(run_dir / "summary.json", {"mission": mission["name"], "mission_file": args.mission_file, "iterations": iterations}) if summary.get("abort_mission"): - log("on_step_failure=abort_mission — stopping remaining iterations") + log("stopping remaining iterations" + + (" (manual stop)" if summary["status"] == "stopped" else + " (on_step_failure=abort_mission)")) break passed = sum(1 for s in iterations if s["status"] == "passed") diff --git a/simulation/isaac-sim/extensions/PegasusSimulator b/simulation/isaac-sim/extensions/PegasusSimulator index fe8b5a101..c94a4fb1f 160000 --- a/simulation/isaac-sim/extensions/PegasusSimulator +++ b/simulation/isaac-sim/extensions/PegasusSimulator @@ -1 +1 @@ -Subproject commit fe8b5a101857f2cda290b9b677b3a95c4cca6b09 +Subproject commit c94a4fb1f1a2abff03e28f6d42ee47ca705d10cb From 12a7240c3379fecd9e4370546df2e9775cacc2bd Mon Sep 17 00:00:00 2001 From: krrishj18 Date: Tue, 21 Jul 2026 15:58:59 -0400 Subject: [PATCH 10/27] config updates --- osmo/missions/example_takeoff_land.yaml | 78 ++++++++++++++----- .../isaac-sim/docker/docker-compose.yaml | 9 +++ .../example_multi_drone_scene_import.py | 44 +++++++++-- 3 files changed, 107 insertions(+), 24 deletions(-) diff --git a/osmo/missions/example_takeoff_land.yaml b/osmo/missions/example_takeoff_land.yaml index e64bf7892..237447b12 100644 --- a/osmo/missions/example_takeoff_land.yaml +++ b/osmo/missions/example_takeoff_land.yaml @@ -1,4 +1,5 @@ -# Example mission: takeoff → hover 30s → land, 3 times, Isaac Sim headless. +# Example mission: takeoff → hover 30s → land, cycled across two Isaac Sim +# scenes (headless), one iteration per scene. # # Run on OSMO: # airstack osmo:mission osmo/missions/example_takeoff_land.yaml --pool @@ -33,12 +34,51 @@ env: NUM_ROBOTS: 1 ISAAC_SIM_HEADLESS: "true" ISAAC_SIM_USE_STANDALONE: "true" - # The default script only spawns one drone; for NUM_ROBOTS > 1 switch to - # example_multi_px4_pegasus_launch_script.py. - ISAAC_SIM_SCRIPT_NAME: example_one_px4_pegasus_launch_script.py + # Reads the per-environment overrides (ENV_URL, STAGE_SCALE, SPAWN_CONFIGS, + # SPAWN_HEIGHT_M) exported from `environments:` below. Without environments, + # use example_one_px4_pegasus_launch_script.py for a single default drone. + ISAAC_SIM_SCRIPT_NAME: example_multi_drone_scene_import.py # Missions are unattended — the sim must start playing without a GUI click. PLAY_SIM_ON_START: "true" +# Environments cycled across iterations (optional — drop this list to run +# every iteration in the launch script's default scene). +# +# UPPERCASE keys are exported as env vars before `airstack up`, so the +# isaac-sim launch script picks them up: +# ENV_URL: full omniverse://… URL, or just the path after the nucleus +# server (the launch script prepends omniverse:///) +# STAGE_SCALE: stage meters-per-unit scale factor +# SPAWN_CONFIGS: fixed per-drone spawn list; only x_m / y_m required +# (z_m, orient, lidar_min_range default per drone) +# SPAWN_HEIGHT_M: default spawn height above the floor +# +# lowercase keys are per-environment step arguments, referenced from any step +# via {{env.}} — e.g. takeoff_altitude_m below sets a different takeoff +# height per scene. +environments: + - name: fireacademy + ENV_URL: "Projects/AirStack/scenes/urban/allegheny_county_fire_academy/fire_academy.scene.usd" + STAGE_SCALE: "0.01" + SPAWN_CONFIGS: + - {x_m: 32.0, y_m: 12.6, orient: [0.0, 0.0, -0.937, 0.35]} + SPAWN_HEIGHT_M: "0.03" + takeoff_altitude_m: 10.0 + + - name: retroneighborhood + ENV_URL: "Library/Stages/RetroNeighborhood/RetroNeighborhood.stage.usd" + STAGE_SCALE: "0.01" + SPAWN_CONFIGS: + - {x_m: 6.764, y_m: -1.767, orient: [0.0, 0.0, -0.1193, 0.9929]} + SPAWN_HEIGHT_M: "0.3" + # Taller buildings — take off higher in this scene. + takeoff_altitude_m: 20.0 + +# round_robin(default): iteration i runs environments[(i-1) % n], so +# `iterations` is the TOTAL count — 2 here = one run per scene. +# grouped: each environment runs `iterations` times in a row. +environment_order: grouped + # Where `action` steps are sent by default: # gcs — through the GCS action_relay, the same String-JSON → typed-goal # path Foxglove and the GCS panels use (exercises the full @@ -47,8 +87,9 @@ env: # Override per step with `via:`. command_route: gcs -# Full up → fly → collect → down cycles. -iterations: 3 +# Full up → fly → collect → down cycles (total across environments — see +# environment_order above). +iterations: 2 # Gate before any steps run: per robot, MAVROS connected, then # local_position/odom publishing (PX4 EKF converged = ready to arm). @@ -93,24 +134,25 @@ on_step_failure: abort_iteration steps: - action: task: takeoff # → //tasks/takeoff (TakeoffTask) - goal: {target_altitude_m: 10.0, velocity_m_s: 1.0} - #to specify robot(s) - #robots: [1] (1st robot) - #robots: [1, 3] (1st and 3rd robot) - #robots: all (all robots) + # {{env.takeoff_altitude_m}} resolves per iteration from the current + # `environments:` entry above. + goal: {target_altitude_m: "{{env.takeoff_altitude_m}}", velocity_m_s: 1.0} + # robot selection (default all): + #robots: [1] + #robots: [1, 3] + #robots: all - wait: 30 # hover - run: # arbitrary command, that runs per robot cmd: ros2 topic echo --once /{robot}/odometry_conversion/odometry timeout_s: 20 - #to specify robot(s): - #robots: [1] (1st robot) - #robots: [1, 3] (1st and 3rd robot) - - #To specify the container: - #container: gcs (GCS container) - #container: + # robot selection (default all): + #robots: [1] + #robots: [1, 3] + # container to run in (default robot_{n}): + #container: gcs + #container: - action: task: land # → //tasks/land (LandTask) diff --git a/simulation/isaac-sim/docker/docker-compose.yaml b/simulation/isaac-sim/docker/docker-compose.yaml index dfd699aa4..d2bd530ed 100644 --- a/simulation/isaac-sim/docker/docker-compose.yaml +++ b/simulation/isaac-sim/docker/docker-compose.yaml @@ -42,6 +42,15 @@ services: - NUM_ROBOTS=${NUM_ROBOTS:-1} - ENABLE_LIDAR=${ENABLE_LIDAR:-false} - ISAAC_SIM_HEADLESS=${ISAAC_SIM_HEADLESS:-false} + # Per-environment scene + spawn overrides (set by the mission runner per + # iteration from a mission's `environments:` entry; read by + # example_multi_drone_scene_import.py). Empty by default → the launch + # script keeps its hardcoded fallbacks. Without this passthrough the + # launch script never sees them. + - ENV_URL=${ENV_URL:-} + - STAGE_SCALE=${STAGE_SCALE:-} + - SPAWN_CONFIGS=${SPAWN_CONFIGS:-} + - SPAWN_HEIGHT_M=${SPAWN_HEIGHT_M:-} # Pegasus physics tuning — read by pegasus/simulator/params.py - PX4_PHYSICS_HZ=${PX4_PHYSICS_HZ:-100} - ARDUPILOT_PHYSICS_HZ=${ARDUPILOT_PHYSICS_HZ:-800} diff --git a/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py b/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py index 89fbe1865..d094d3cd8 100644 --- a/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py +++ b/simulation/isaac-sim/launch_scripts/example_multi_drone_scene_import.py @@ -12,6 +12,7 @@ "omniverse://airlab-nucleus.andrew.cmu.edu/NVIDIA/Assets/Isaac/5.1" ) +import json import os import sys import time @@ -50,7 +51,16 @@ #env/stage path and scale ENV_URL = f"omniverse://{NUCLEUS_SERVER}/Projects/AirStack/scenes/urban/allegheny_county_fire_academy/fire_academy.scene.usd" -STAGE_SCALE = 0.01 +# Per-environment overrides (the mission runner exports these per iteration +# from a mission's `environments:` entry — see osmo/missions/README.md); +# the hardcoded values above/below stay the standalone-run fallbacks. +# ENV_URL accepts a full URL or just the path after the nucleus server. +_env_url_override = os.environ.get("ENV_URL") +if _env_url_override: + ENV_URL = (_env_url_override if "://" in _env_url_override + else f"omniverse://{NUCLEUS_SERVER}/{_env_url_override.lstrip('/')}") + +STAGE_SCALE = float(os.environ.get("STAGE_SCALE") or 0.01) DRONE_USD = "~/.local/share/ov/data/documents/Kit/shared/exts/pegasus.simulator/pegasus/simulator/assets/Robots/Iris/iris.usd" @@ -73,13 +83,35 @@ # {"domain_id": 1, "x_m": 20.0, "y_m": -7.0, ...} # {"domain_id": 2, "x_m": 17.0, "y_m": 1.5, ...} -SPAWN_HEIGHT_ABOVE_FLOOR_M = 0.03 -DRONE_CONFIGS = [ - {"domain_id": 1, "x_m": 32.0, "y_m": 12.6, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": 0.75}, - {"domain_id": 2, "x_m": 28.0, "y_m": 14.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": 0.75}, - {"domain_id": 3, "x_m": 32.0, "y_m": 19.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": 0.75} +SPAWN_HEIGHT_ABOVE_FLOOR_M = float(os.environ.get("SPAWN_HEIGHT_M") or 0.03) +LIDAR_MIN_RANGE_M = 0.75 + +_DEFAULT_DRONE_CONFIGS = [ + {"domain_id": 1, "x_m": 32.0, "y_m": 12.6, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": LIDAR_MIN_RANGE_M}, + {"domain_id": 2, "x_m": 28.0, "y_m": 14.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": LIDAR_MIN_RANGE_M}, + {"domain_id": 3, "x_m": 32.0, "y_m": 19.8, "z_m": SPAWN_HEIGHT_ABOVE_FLOOR_M, "orient": [0.0, 0.0, -0.937, 0.35], "lidar_min_range": LIDAR_MIN_RANGE_M} ] +# Spawn-location override: JSON list of dicts, one per drone. Only x_m / y_m +# are required — domain_id, z_m, orient, lidar_min_range default per drone. +# SPAWN_CONFIGS='[{"x_m": 32.0, "y_m": 12.6, "orient": [0, 0, -0.937, 0.35]}]' +_SPAWN_CONFIGS = os.environ.get("SPAWN_CONFIGS") +if _SPAWN_CONFIGS: + DRONE_CONFIGS = json.loads(_SPAWN_CONFIGS) + for _i, _c in enumerate(DRONE_CONFIGS, start=1): + _c.setdefault("domain_id", _i) + _c.setdefault("z_m", SPAWN_HEIGHT_ABOVE_FLOOR_M) + _c.setdefault("orient", [0.0, 0.0, 0.0, 1.0]) + _c.setdefault("lidar_min_range", LIDAR_MIN_RANGE_M) +else: + DRONE_CONFIGS = _DEFAULT_DRONE_CONFIGS + +# Logged so each iteration's chosen environment + layout is captured in the +# isaac-sim container logs, which mission_runner snapshots per iteration. +print(f"[spawn] ENV_URL={ENV_URL}", flush=True) +print(f"[spawn] STAGE_SCALE={STAGE_SCALE}", flush=True) +print(f"[spawn] DRONE_CONFIGS={json.dumps(DRONE_CONFIGS)}", flush=True) + # Top-down "map" camera. Captures one aerial of the static scene that the # GCS visualizer turns into a textured ground in Foxglove's 3D panel. The # camera centers on (OVERHEAD_CENTER_X_M, OVERHEAD_CENTER_Y_M) in world From ee1c269a3ce9cc2a417847a5353703dcc856b809 Mon Sep 17 00:00:00 2001 From: pvkumara Date: Sun, 26 Jul 2026 20:54:13 -0400 Subject: [PATCH 11/27] ci(orchestrator): migrate ephemeral CI runners from OpenStack to NVIDIA OSMO Replace the OpenStack-Nova spawn/reap backend with OSMO workflow submission. The GitHub side is unchanged (self-hosted/airstack-ephemeral labels, single-use JIT runner tokens, same-repo fork guard) and the one-job-per-worker destroy-after model is preserved; only the spawn target moved from creating a Nova VM to submitting an OSMO workflow. orchestrator.py: submit/query/cancel/list via the osmo CLI, job_id -> workflow_id state, re-login-on-auth-failure, orphan sweep via osmo workflow list; drop floating-IP/boot-volume/placement/keypair/security-group logic. runner.Dockerfile + runner-entrypoint.sh + runner-workflow.yaml.j2: prebaked privileged docker-in-docker + GPU GitHub runner image/task (replaces cloud-init.yaml.j2). config.example.yaml, setup.sh, airstack-orchestrator.service, requirements.txt: OSMO service-account token auth, install the osmo CLI, drop openstacksdk. Docs (AGENTS.md, tests/README.md, orchestrator README) updated to OSMO. Co-authored-by: Cursor --- .github/orchestrator/README.md | 274 +++---- .../airstack-orchestrator.service | 17 +- .github/orchestrator/cloud-init.yaml.j2 | 71 -- .github/orchestrator/config.example.yaml | 136 ++-- .github/orchestrator/orchestrator.py | 768 ++++++++---------- .github/orchestrator/requirements.txt | 1 - .github/orchestrator/runner-entrypoint.sh | 39 + .github/orchestrator/runner-workflow.yaml.j2 | 48 ++ .github/orchestrator/runner.Dockerfile | 62 ++ .github/orchestrator/setup.sh | 51 +- AGENTS.md | 14 +- tests/README.md | 39 +- 12 files changed, 766 insertions(+), 754 deletions(-) delete mode 100644 .github/orchestrator/cloud-init.yaml.j2 create mode 100644 .github/orchestrator/runner-entrypoint.sh create mode 100644 .github/orchestrator/runner-workflow.yaml.j2 create mode 100644 .github/orchestrator/runner.Dockerfile diff --git a/.github/orchestrator/README.md b/.github/orchestrator/README.md index c10da3383..a1f0d7e94 100644 --- a/.github/orchestrator/README.md +++ b/.github/orchestrator/README.md @@ -1,131 +1,117 @@ -# AirStack CI Orchestrator +# AirStack CI Orchestrator (OSMO backend) -This describes how to use a self-hosted OpenStack VM to run GitHub Actions jobs on truly ephemeral workers. The orchestrator is a Python service that continuously polls GitHub for queued workflow jobs, spawns a fresh OpenStack instance for each one with a single-use JIT runner token, and reaps (deletes) the instance when the job completes. This allows us to run CI workloads on GPU-equipped VMs without sharing any state between runs or exposing long-lived credentials on the worker. +This describes how a small always-on orchestrator service runs GitHub Actions jobs on truly ephemeral GPU workers scheduled by [NVIDIA OSMO](https://nvidia.github.io/OSMO/). The orchestrator is a Python service that continuously polls GitHub for queued workflow jobs, submits a fresh **OSMO workflow** for each one (a single-use JIT runner in a privileged, GPU-enabled container), and reaps it when the job completes. Each CI job runs on a clean pod with no state shared between runs and no long-lived credentials on the worker. -The orchestrator VM is the only host that holds the GitHub PAT and the OpenStack credential; the workers are destroyed after a single job. +This is a drop-in replacement for the previous OpenStack-Nova backend. The GitHub side is unchanged — `system-tests.yml` still uses `runs-on: [self-hosted, airstack-ephemeral]`, the single-use JIT runner config, and the same-repo fork guard. Only the *spawn target* changed from "create a Nova VM" to "submit an OSMO workflow", so the one-job-per-worker, destroy-after semantics are identical: when the runner's `run.sh` exits after one job, the OSMO task completes and the pod is torn down. + +The orchestrator host is the only machine that holds the GitHub PAT and the OSMO service-account token; workers are destroyed after a single job. ## Architecture ``` ┌─────────────────────────────────────────────────────────────┐ -│ Orchestrator VM (airstack-ci-cd-orchestrator) │ +│ Orchestrator host (airstack-ci-cd-orchestrator, no GPU) │ │ │ │ airstack-orchestrator.service → orchestrator.py │ │ spawn loop (every 15s): │ │ • GET /repos//actions/runs?status=queued │ │ • POST /repos//actions/runners/generate-jitconfig│ -│ • openstack server create (image, flavor, user_data) │ -│ • record (job_id → server_id) in state.json │ +│ • osmo workflow submit runner-workflow.yaml --pool ... │ +│ • record (job_id → workflow_id) in state.json │ │ reap loop (every 30s): │ -│ • job completed → openstack server delete │ -│ • job age > N min → force delete (straggler) │ -│ • owned but not in state → orphan reap │ +│ • job completed → osmo workflow cancel (if live) │ +│ • job age > N min → osmo workflow cancel (straggler) │ +│ • our-named but not in state → orphan cancel │ │ │ │ /etc/airstack-orchestrator/ │ │ config.yaml │ │ github-pat │ -│ /home/orchestrator/.config/openstack/clouds.yaml │ +│ osmo-token (OSMO service-account token) │ │ /var/lib/airstack-orchestrator/state.json │ +│ /var/lib/airstack-orchestrator/.config/osmo (CLI session) │ └─────────┬─────────────────────────────────┬─────────────────┘ - │ Nova / Neutron API │ GitHub REST API + │ osmo CLI (submit/query/cancel) │ GitHub REST API ▼ ▼ ┌──────────────────────────────────┐ ┌──────────────────────┐ -│ Ephemeral worker (per job) │ │ GitHub Actions │ -│ Image: Ubuntu-24.04-GPU-Headless│ │ workflow_job queue │ -│ cloud-init: │ └──────────────────────┘ -│ install docker + nv toolkit │ -│ download GH runner │ +│ OSMO CI GPU pool (privileged) │ │ GitHub Actions │ +│ Ephemeral runner pod (per job): │ │ workflow_job queue │ +│ Image: airstack-ci-runner │ └──────────────────────┘ +│ start dockerd (DinD) │ │ run.sh --jitconfig │ -│ shutdown -h +1 │ +│ exit → task done → pod reaped │ └──────────────────────────────────┘ ``` Key properties: -- **Truly ephemeral**: every job runs on a clean VM. No Docker layer cache pollution, no leftover networks, no carry-over from prior runs. +- **Truly ephemeral**: every job runs on a clean pod. No Docker layer cache pollution, no leftover containers, no carry-over from prior runs. - **PAT isolation**: the GitHub PAT lives only on the orchestrator. Workers receive a single-use [JIT runner config](https://docs.github.com/en/rest/actions/self-hosted-runners?apiVersion=2022-11-28#create-configuration-for-a-just-in-time-runner-for-a-repository) — a base64 token bound to one runner registration, valid only for a short window. -- **Application-credential auth**: the orchestrator authenticates to OpenStack with an application credential (revocable, scoped, no password), not the user's `openrc.sh`. -- **Crash-safe reaping**: every server we spawn is tagged with `airstack-role=ephemeral-runner`. The reap loop force-deletes any owned server not present in `state.json`, so a crashed orchestrator can't leak instances. +- **Service-account auth**: the orchestrator authenticates to OSMO with a shared, non-personal [service-account token](https://nvidia.github.io/OSMO/main/deployment_guide/appendix/authentication/service_accounts.html) (the analog of the old OpenStack application credential). CI runs never route through an individual's account, so PRs don't consume anyone's personal GPU quota and nothing breaks when a person leaves. +- **Crash-safe reaping**: every workflow is named `gha-runner--`. The reap loop cancels any active workflow with that prefix not present in `state.json`, so a crashed orchestrator can't leak workflows. ## Prerequisites -- OpenStack instance already setup for the orchestrator VM. The orchestrator itself is lightweight and doesn't need a GPU. 1 vCPU, 2GB RAM, and 20GB disk is sufficient for the orchestrator service. Make sure you can ssh into it and that it has outbound internet access. -- An OpenStack flavor with GPU passthrough and enough disk to run Docker + the tests. The orchestrator spawns workers from this flavor, so it must have a GPU and sufficient disk (or `boot_volume_size_gb` must be set) to run the workloads. It's common for GPU flavors to have `disk=0`, which means they boot from an ephemeral disk — in that case, you must set `boot_volume_size_gb` to a value large enough for the OS + Docker images + test assets (e.g., 40GB). If your OpenStack setup supports it, you can also boot from a Cinder volume sourced from an image; in that case, pre-bake Docker and the NVIDIA toolkit into the image to speed up boot time. + +- **Orchestrator host** — a small always-on VM (no GPU). 1 vCPU, 2GB RAM, 20GB disk, outbound internet to `api.github.com` and your OSMO URL. This is the only long-lived piece; the GPU compute is ephemeral pods on OSMO. +- **An OSMO service account + dedicated CI pool.** Ask your OSMO admin to: + 1. Create a service account (e.g. `svc-airstack-ci`) and a long-lived access token — `osmo user create` + `osmo token set`. On IdP-backed deployments (e.g. auth tied to the CMU Andrew directory) this is a non-personal identity, so it survives people graduating/leaving. If policy forbids OSMO-native service accounts, use a *functional/departmental* identity, never a personal one. + 2. Grant that account a role whose policy allows `workflow:Create/Cancel/Query` **scoped to a dedicated CI GPU pool** (e.g. `pool/airstack-ci`) that has its own allocation, so CI doesn't contend with researchers' interactive jobs. + 3. **Enable "Privileged Mode Allowed"** on that pool's platform. The AirStack tests run `airstack up` (docker compose) inside the worker, which requires an inner Docker daemon → a privileged container. Without this, submissions are rejected. + 4. Confirm the API gateway (Envoy) accepts OSMO access tokens for the API (not only interactive IdP logins). +- **A prebaked runner image** pushed to a registry the pool can pull (see below). ## One-time setup -### 1. Create OpenStack application credential +### 1. Build & push the runner image -On your local workstation (not the orchestrator VM): +The worker image bakes in Docker CE + compose, the NVIDIA container toolkit, and the GitHub Actions runner (what cloud-init used to install at boot on the VM), so pod start is fast and the JIT token can't expire mid-bootstrap. ```bash -source ~/.airlabcloud/openrc.sh -openstack application credential create airstack-orchestrator \ - --description "AirStack CI orchestrator — spawns ephemeral test runners" +cd .github/orchestrator +docker build -f runner.Dockerfile \ + --build-arg RUNNER_VERSION=2.334.0 \ + -t /airstack-ci-runner:2.334.0 . +docker push /airstack-ci-runner:2.334.0 ``` -The output prints `id` and `secret`. Build a `clouds.yaml`: - -```yaml -clouds: - airstack: - auth_type: v3applicationcredential - auth: - auth_url: https://airlab-cloud.andrew.cmu.edu:5000/v3/ - application_credential_id: - application_credential_secret: - region_name: Airlab - interface: public - identity_api_version: 3 -``` +Set `runner_image: /airstack-ci-runner:2.334.0` in `config.yaml` (step 4). Keep `RUNNER_VERSION` in sync with an [actions/runner release](https://github.com/actions/runner/releases). -### 2. Stage credentials on the orchestrator VM +### 2. Stage credentials on the orchestrator host ```bash -# clouds.yaml: install for the orchestrator user (created in step 3) -scp clouds.yaml ubuntu@:/tmp/clouds.yaml - # GitHub PAT: needs `Actions: read/write` and `Administration: read/write` # (fine-grained) or classic `repo` scope. -scp ~/.airlabcloud/airstack-github-pat.txt \ - ubuntu@:/tmp/github-pat +scp ~/airstack-github-pat.txt ubuntu@:/tmp/github-pat + +# OSMO service-account token (from `osmo token set`, provided by your admin). +scp ~/svc-airstack-ci-token.txt ubuntu@:/tmp/osmo-token ``` ### 3. Run setup.sh -On the orchestrator VM: +On the orchestrator host: ```bash git clone https://github.com/castacks/AirStack.git /tmp/airstack sudo bash /tmp/airstack/.github/orchestrator/setup.sh ``` -`setup.sh` creates the `orchestrator` system user, builds the Python venv, copies `orchestrator.py` and `cloud-init.yaml.j2` into `/opt/airstack-orchestrator/`, scaffolds `/etc/airstack-orchestrator/`, installs the systemd unit, and consumes `/tmp/github-pat`. - -You still need to put the `clouds.yaml` in place under the orchestrator user's home: - -```bash -sudo install -d -o orchestrator -g orchestrator -m 0700 \ - /home/orchestrator/.config/openstack -sudo install -o orchestrator -g orchestrator -m 0600 \ - /tmp/clouds.yaml /home/orchestrator/.config/openstack/clouds.yaml -sudo shred -u /tmp/clouds.yaml -``` +`setup.sh` creates the `orchestrator` system user, installs the `osmo` CLI, builds the Python venv, copies `orchestrator.py` and `runner-workflow.yaml.j2` into `/opt/airstack-orchestrator/`, scaffolds `/etc/airstack-orchestrator/`, installs the systemd unit, and consumes `/tmp/github-pat` and `/tmp/osmo-token`. ### 4. Fill in `/etc/airstack-orchestrator/config.yaml` -Edit the placeholders the example ships with: - | Field | What goes here | How to find it | |------|---------------|----------------| -| `flavor_name` | OpenStack flavor with GPU + enough disk | `openstack flavor list` | -| `network_name` | Network the workers attach to | `openstack network list` | -| `keypair_name` | SSH keypair for break-glass access | `openstack keypair list` | -| `security_group` | Outbound 443 must be allowed | `openstack security group list` | -| `availability_zone` | Optional AZ for the spawned instance; leave empty to let Nova pick | `openstack availability zone list` | -| `boot_volume_size_gb` | Set >0 if your flavor has `disk=0` (common for GPU flavors) — boots from a Cinder volume of this size sourced from `image_id`; leave 0 for direct image-boot | `openstack flavor show ` (check disk field) | -| `floating_ips` | Pre-allocated FIP pool, rotated through sequentially — each spawn picks the first free one. `max_concurrent` is capped at `len(pool)`. Leave empty to skip FIP attachment | `openstack floating ip list` | +| `osmo_url` | Your OSMO web service URL | from your OSMO admin | +| `pool` | Dedicated CI GPU pool | `osmo pool list` / the OSMO UI | +| `platform` | Optional hardware type within the pool; empty = pool default | `osmo pool list` | +| `runner_image` | Image from step 1 | the registry you pushed to | +| `cpu` / `gpu` / `memory` / `storage` | Resource request for the worker | size for full stack + sim | +| `privileged` | Must be `true` (docker compose inside the pod) | — | +| `priority` | `HIGH` \| `NORMAL` \| `LOW` | — | | `repo` | `owner/name` of the repo to poll | from GitHub URL | -| `runner_version` | Version tag from [actions/runner releases](https://github.com/actions/runner/releases) | check before each major upgrade | +| `runner_version` | Runner version baked into `runner_image` | matches step 1 | +| `max_concurrent` | Max simultaneous in-flight workflows | — | +| `max_job_minutes` | Straggler cancel ceiling | exceed the longest job | ### 5. Start the service @@ -134,7 +120,7 @@ sudo systemctl enable --now airstack-orchestrator.service journalctl -u airstack-orchestrator.service -f ``` -You should see `orchestrator started: repo=... labels=... max_concurrent=N` and then periodic poll activity. +You should see `orchestrator started (OSMO backend): repo=... pool=... max_concurrent=N`, an `osmo login succeeded` line, and then periodic poll activity. ## End-to-end verification @@ -142,147 +128,121 @@ You should see `orchestrator started: repo=... labels=... max_concurrent=N` and # Trigger a fast build-only run. gh workflow run system-tests.yml -f marks=build_docker -# Within ~30s, a server should appear: -openstack server list --metadata airstack-role=ephemeral-runner -# or if your OpenStack setup doesn't support metadata queries: -openstack server list --name '^ephemeral-' +# Within ~30s, a workflow should appear in the CI pool: +osmo workflow list --name gha-runner- --pool airstack-ci # Watch GitHub → Actions → Runners — the ephemeral runner should appear, # pick up the job, then disappear. -# Within ~30s of job completion, the server should be gone: -openstack server list --metadata airstack-role=ephemeral-runner -openstack server list --name '^ephemeral-' +# Within ~30s of job completion, the workflow should be terminal / gone from +# the active list: +osmo workflow list --name gha-runner- --pool airstack-ci --status RUNNING PENDING WAITING ``` ## Operational notes -- **State file**: `/var/lib/airstack-orchestrator/state.json` is the in-flight job tracker. Wiping it triggers an orphan sweep on the next reap iteration — owned servers will be force-deleted. Don't wipe it while jobs are mid-flight unless that's what you want. -- **Stuck instance**: any server older than `max_job_minutes` (default 90) is force-deleted regardless of GitHub job status. Bump this if liveliness/autonomy runs grow longer than ~75 minutes. +- **State file**: `/var/lib/airstack-orchestrator/state.json` is the in-flight job tracker (`job_id → workflow_id`). Wiping it triggers an orphan sweep on the next reap iteration — active `gha-runner-*` workflows will be cancelled. Don't wipe it while jobs are mid-flight unless that's what you want. +- **Straggler**: any workflow whose job has run longer than `max_job_minutes` (default 48h) is force-cancelled regardless of GitHub job status. +- **OSMO token rotation** (tokens expire — default 31 days): mint a new one and restart. + ```bash + # (admin) osmo token set svc-airstack-ci-token-2 --user svc-airstack-ci \ + # --roles osmo-user --expires-at 2027-12-31 + sudo install -o root -g orchestrator -m 0640 /tmp/osmo-token /etc/airstack-orchestrator/osmo-token + sudo systemctl restart airstack-orchestrator.service # re-runs `osmo login` + ``` - **PAT rotation**: `sudo install -o root -g orchestrator -m 0640 /tmp/new-pat /etc/airstack-orchestrator/github-pat && sudo systemctl restart airstack-orchestrator.service`. -- **Pause spawning** (e.g. for maintenance): `sudo systemctl stop airstack-orchestrator.service`. Already-spawned workers will still complete their jobs and self-shutdown; on restart, the reap loop deletes them. -- **Logs**: `journalctl -u airstack-orchestrator.service -f`. Cloud-init logs from individual workers are visible only via `openstack console log show ` while the worker is running. +- **Pause spawning** (e.g. for maintenance): `sudo systemctl stop airstack-orchestrator.service`. Already-submitted workers still complete their jobs; on restart, the reap loop cleans up. +- **Logs**: `journalctl -u airstack-orchestrator.service -f`. Per-worker logs come from `osmo workflow logs `. ## Debugging a failed job -When a GitHub workflow run fails or stalls, the failure can be in any of four places: the orchestrator (didn't spawn), cloud-init (didn't bootstrap), the GH Actions runner (didn't register or crashed), or the workflow steps themselves. Each has a different inspection path. +When a GitHub workflow run fails or stalls, the failure can be in one of four places: the orchestrator (didn't submit), the OSMO task (didn't schedule/pull), the GH Actions runner (didn't register or crashed), or the workflow steps themselves. Each has a different inspection path. -### 1. Find which worker ran the job +### 1. Find which workflow ran the job -`state.json` is the authoritative job ↔ server ↔ floating-IP map: +`state.json` is the authoritative job ↔ workflow map: ```bash -sudo jq -r '.jobs | to_entries[] | "\(.key)\t\(.value.server_id)\t\(.value.floating_ip)\t\(.value.runner_name)"' \ +sudo jq -r '.jobs | to_entries[] | "\(.key)\t\(.value.workflow_id)\t\(.value.workflow_name)"' \ /var/lib/airstack-orchestrator/state.json ``` -Pick the row for your failing `job_id` (visible in the GitHub Actions URL). Save the values: +Pick the row for your failing `job_id` (visible in the GitHub Actions URL): ```bash JOB_ID=73286176852 # from the GitHub UI -SERVER=$(sudo jq -r ".jobs[\"$JOB_ID\"].server_id" /var/lib/airstack-orchestrator/state.json) -FIP=$( sudo jq -r ".jobs[\"$JOB_ID\"].floating_ip" /var/lib/airstack-orchestrator/state.json) +WF=$(sudo jq -r ".jobs[\"$JOB_ID\"].workflow_id" /var/lib/airstack-orchestrator/state.json) ``` -If the job isn't in `state.json`, the orchestrator never spawned for it — see step 2 below. +If the job isn't in `state.json`, the orchestrator never submitted for it — see step 2. -### 2. Did the orchestrator spawn at all? +### 2. Did the orchestrator submit at all? ```bash sudo journalctl -u airstack-orchestrator.service --since "30 min ago" --no-pager ``` -What you want to see for a healthy spawn: +Healthy submit looks like: ```text -spawned server for job () -attached floating IP to server (job ) +submitted workflow for job () ``` -Common things that block a spawn (and how to spot them): +Common things that block a submit (and how to spot them): | Log line / symptom | What it means | Fix | |---|---|---| -| `find_queued_jobs failed: 401 ...` | PAT expired / wrong scope | Rotate the PAT (see Operational notes) | -| `spawn failed for job ...: Block Device Mapping is Invalid` | Flavor has `disk=0` and `boot_volume_size_gb` is 0 | Set `boot_volume_size_gb > 0` | -| `no free floating IP in pool` | All FIPs in `floating_ips` are already in use | Wait for an in-flight job to complete, or expand the pool | -| `floating_ips configured but not found` | Pool addresses don't exist in the project | Double-check `openstack floating ip list` | -| Job is queued in GitHub but no `spawned` log | Runner labels in the workflow's `runs-on` don't match `runner_labels` in config | Make them match | - -### 3. SSH into a running worker +| `find_queued_jobs failed: 401 ...` | GitHub PAT expired / wrong scope | Rotate the PAT | +| `osmo login failed ...` / `auth error` | OSMO token expired/invalid, or Envoy rejects access tokens | Rotate the OSMO token; confirm gateway accepts access tokens | +| `osmo workflow submit failed ... privileged` | Pool platform doesn't allow privileged | Ask admin to enable "Privileged Mode Allowed" on the CI pool | +| `osmo workflow submit failed ... pool` / permission | Service-account role lacks `workflow:Create` on the pool | Fix the role's pool-scoped policy | +| Job queued in GitHub but no `submitted` log | `runs-on` labels don't match `runner_labels` | Make them match | -If the worker is `ACTIVE`, the floating IP is attached and you can connect directly. The keypair was injected during spawn — use the matching private key: +### 3. Inspect the workflow / worker ```bash -ssh -i .pem ubuntu@"$FIP" -``` +# Status and scheduling detail. +osmo workflow query "$WF" --verbose -If your workstation can't reach the FIP subnet, jump through the orchestrator (which is on the same network): +# Scheduling / lifecycle events (image pull, start, evict, ...). +osmo workflow events "$WF" --task runner -```bash -ssh -J ubuntu@ -i .pem ubuntu@"$FIP" +# Combined stdout of the runner task — shows dockerd start, run.sh, and the job. +osmo workflow logs "$WF" --task runner +osmo workflow logs "$WF" --task runner --error # error stream +osmo workflow logs "$WF" --task runner -n 300 # last 300 lines ``` -### 4. SSH into a SHUTOFF worker +### 4. Break-glass shell into a running worker -Workers shut themselves down after `run.sh` exits (whether the job succeeded, failed, or the runner crashed). The orchestrator only deletes a server once GitHub reports the job `completed`, so a SHUTOFF worker is preserved while you debug. +If the workflow is still `RUNNING`, exec into the pod (replaces the old SSH-via-floating-IP path): ```bash -# Optional but safer — keep the orchestrator from reaping mid-session. -sudo systemctl stop airstack-orchestrator.service - -openstack server start "$SERVER" -# Wait ~30s, then SSH using the FIP from state.json. -ssh -i .pem ubuntu@"$FIP" +osmo workflow exec "$WF" runner # /bin/bash in the runner task ``` -When done, delete the worker manually and resume the orchestrator: +Once inside: ```bash -openstack server delete "$SERVER" -sudo jq "del(.jobs[\"$JOB_ID\"])" /var/lib/airstack-orchestrator/state.json \ - | sudo tee /var/lib/airstack-orchestrator/state.json.new >/dev/null -sudo mv /var/lib/airstack-orchestrator/state.json.new /var/lib/airstack-orchestrator/state.json -sudo systemctl start airstack-orchestrator.service -``` - -### 5. What to read once you're on the worker +# GitHub Actions runner diagnostics. +ls -lt /home/runner/actions-runner/_diag/ +tail -300 /home/runner/actions-runner/_diag/Runner_*.log +tail -300 /home/runner/actions-runner/_diag/Worker_*.log -```bash -# Combined boot + cloud-init output. Most useful single file: shows every -# line our airstack-runner-bootstrap.sh printed, including run.sh's exit. -sudo less /var/log/cloud-init-output.log -sudo tail -300 /var/log/cloud-init-output.log - -# Cloud-init's structured log — quick way to surface errors. -sudo grep -E 'WARN|ERROR|FAIL' /var/log/cloud-init.log - -# GitHub Actions runner diagnostics. The Worker_*.log corresponds to the -# actual job execution; Runner_*.log covers registration and dispatch. -ls -lt /home/ubuntu/actions-runner/_diag/ -sudo tail -300 /home/ubuntu/actions-runner/_diag/Runner_*.log -sudo tail -300 /home/ubuntu/actions-runner/_diag/Worker_*.log - -# Sanity-check Docker came up cleanly — a frequent failure point. -sudo systemctl status docker +# Inner Docker daemon (a frequent failure point for `airstack up`). +cat /var/log/dockerd.log docker info 2>&1 | head +nvidia-smi ``` -### 6. Console log fallback - -Some flavors on this cloud don't expose the serial console (`openstack console log show` returns *Guest does not have a console available*). For those, the SSH path above is the only option. Where it does work, the console log persists across SHUTOFF and is faster than restarting the VM: - -```bash -openstack console log show "$SERVER" | tail -200 -``` - -### 7. Common failure patterns at the worker +### 5. Common failure patterns at the worker -| Symptom in `cloud-init-output.log` (near end) | Cause | Fix | +| Symptom in `osmo workflow logs` | Cause | Fix | |---|---|---| -| `Could not connect to api.github.com` / DNS errors | Security group blocking egress, or no NAT for the network | Allow outbound 443; if behind NAT, ensure FIP networking covers egress | -| `Bad credentials` / `Invalid configuration ... runnerEvent` | JIT config TTL elapsed before `run.sh` started — bootstrap took too long | Pre-bake Docker + nvidia-container-toolkit into the image to shrink bootstrap | -| `nvidia-ctk: command not found` or NVIDIA driver mismatch | Image's driver doesn't match the toolkit version | Use a different image, or pin a compatible toolkit version | -| `apt-get update` fails | Image's apt sources are unreachable from this network | Check network/security-group; or pre-bake packages into the image | -| Runner registered, then `pytest` failed | A normal test failure | Read the GitHub Actions log — that's the canonical view of the workflow output | -| `No space left on device` | `boot_volume_size_gb` too small for Docker images + sim assets | Bump `boot_volume_size_gb` | +| `dockerd did not become ready` | Pod not privileged / DinD blocked | Enable privileged on the pool platform | +| `nvidia-smi unavailable` / no GPU | GPU not requested/passed, or toolkit missing | Check `gpu:` request, platform GPUs, privileged | +| `Could not connect to api.github.com` | Egress blocked from the pool | Allow outbound 443 from the CI pool | +| `Bad credentials` / `Invalid ... runnerEvent` | JIT config TTL elapsed before `run.sh` started | Prebake the image (already done) so start is fast | +| `Cannot connect to the Docker daemon` during tests | inner dockerd crashed | Read `/var/log/dockerd.log` via `osmo workflow exec` | +| Runner registered, then `pytest` failed | A normal test failure | Read the GitHub Actions log — the canonical view | +| `No space left on device` | `storage` too small for images + sim assets | Bump `storage` in `config.yaml` | diff --git a/.github/orchestrator/airstack-orchestrator.service b/.github/orchestrator/airstack-orchestrator.service index 7123232eb..0c5c1843d 100644 --- a/.github/orchestrator/airstack-orchestrator.service +++ b/.github/orchestrator/airstack-orchestrator.service @@ -1,5 +1,5 @@ [Unit] -Description=AirStack CI Orchestrator (spawns ephemeral OpenStack runners) +Description=AirStack CI Orchestrator (submits ephemeral OSMO runner workflows) Documentation=https://github.com/castacks/AirStack/tree/main/.github/orchestrator After=network-online.target Wants=network-online.target @@ -10,17 +10,20 @@ User=orchestrator Group=orchestrator WorkingDirectory=/opt/airstack-orchestrator -# Application credential lives in the orchestrator user's home so openstacksdk -# finds it via the default cloud-config search path. -Environment=HOME=/home/orchestrator -Environment=OS_CLIENT_CONFIG_FILE=/home/orchestrator/.config/openstack/clouds.yaml +# The `osmo` CLI persists its login session under $HOME/XDG dirs. Point them at +# the (writable) state dir so ProtectHome can stay read-only. setup.sh creates +# these directories owned by the orchestrator user. +Environment=HOME=/var/lib/airstack-orchestrator +Environment=XDG_CONFIG_HOME=/var/lib/airstack-orchestrator/.config +Environment=XDG_CACHE_HOME=/var/lib/airstack-orchestrator/.cache +Environment=XDG_STATE_HOME=/var/lib/airstack-orchestrator/.state ExecStart=/opt/airstack-orchestrator/venv/bin/python \ /opt/airstack-orchestrator/orchestrator.py \ --config /etc/airstack-orchestrator/config.yaml \ --pat /etc/airstack-orchestrator/github-pat \ --state /var/lib/airstack-orchestrator/state.json \ - --template /opt/airstack-orchestrator/cloud-init.yaml.j2 + --template /opt/airstack-orchestrator/runner-workflow.yaml.j2 Restart=always RestartSec=10 @@ -33,6 +36,8 @@ KillSignal=SIGTERM NoNewPrivileges=true ProtectSystem=strict ProtectHome=read-only +# ReadWritePaths re-grants write access under ProtectSystem/ProtectHome so the +# OSMO CLI session cache and state.json can be written. ReadWritePaths=/var/lib/airstack-orchestrator PrivateTmp=true diff --git a/.github/orchestrator/cloud-init.yaml.j2 b/.github/orchestrator/cloud-init.yaml.j2 deleted file mode 100644 index 921417c18..000000000 --- a/.github/orchestrator/cloud-init.yaml.j2 +++ /dev/null @@ -1,71 +0,0 @@ -#cloud-config -# Rendered per-spawn by orchestrator.py with two Jinja variables: -# encoded_jit_config - single-use base64 JIT config from GitHub -# runner_version - GitHub Actions runner version (e.g. 2.334.0) -# -# The base image (Ubuntu-24.04-GPU-Headless) already has NVIDIA drivers. -# This cloud-init adds Docker (with the compose plugin), nvidia-container-toolkit, -# downloads the GitHub Actions runner, registers it with the JIT config, runs -# exactly one job (the JIT config + --ephemeral makes the runner exit after one -# job), and shuts the VM down. The orchestrator then deletes the server. - -package_update: true -package_upgrade: false -packages: - - jq - - curl - - ca-certificates - - gnupg - -write_files: - - path: /usr/local/bin/airstack-runner-bootstrap.sh - permissions: "0755" - owner: root:root - content: | - #!/usr/bin/env bash - set -euxo pipefail - - # Install Docker (with compose plugin) from Docker's official channel. - # get.docker.com handles apt repo setup + nvidia-container-toolkit-compatible - # docker-ce, plus the docker-compose-plugin we need for `airstack up`. - curl -fsSL https://get.docker.com | sh - - # nvidia-container-toolkit is required for GPU containers (liveliness / - # autonomy tests). The base image has the NVIDIA *drivers* but we still - # need the container runtime hooks here. - distribution=$(. /etc/os-release; echo "$ID$VERSION_ID") - curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ - | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg - curl -fsSL "https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list" \ - | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ - > /etc/apt/sources.list.d/nvidia-container-toolkit.list - apt-get update - apt-get install -y nvidia-container-toolkit - nvidia-ctk runtime configure --runtime=docker - systemctl restart docker - - usermod -aG docker ubuntu - - # GitHub Actions runner. - RUNNER_VERSION="{{ runner_version }}" - RUNNER_DIR=/home/ubuntu/actions-runner - mkdir -p "$RUNNER_DIR" - cd "$RUNNER_DIR" - curl -fsSL -o runner.tar.gz \ - "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" - tar xzf runner.tar.gz - rm runner.tar.gz - chown -R ubuntu:ubuntu "$RUNNER_DIR" - - # Run exactly one job under the ubuntu user. The JIT config is single-use - # and ephemeral, so run.sh exits after one job completes. - sudo -u ubuntu --preserve-env=HOME -H bash -c \ - "cd '$RUNNER_DIR' && ./run.sh --jitconfig '{{ encoded_jit_config }}'" \ - || echo "runner exited non-zero (job failure or runner error)" - - # Backstop: power down. The orchestrator's reap loop is the authoritative - # deleter — it sees the GitHub job complete and calls Nova delete. - shutdown -h +1 - -runcmd: - - /usr/local/bin/airstack-runner-bootstrap.sh diff --git a/.github/orchestrator/config.example.yaml b/.github/orchestrator/config.example.yaml index 4a47bbe1f..27d8a3975 100644 --- a/.github/orchestrator/config.example.yaml +++ b/.github/orchestrator/config.example.yaml @@ -1,54 +1,63 @@ -# AirStack CI orchestrator configuration. +# AirStack CI orchestrator configuration (OSMO backend). # Copy to /etc/airstack-orchestrator/config.yaml and fill in placeholders. -# --- OpenStack target --- - -# Cloud profile name in ~/.config/openstack/clouds.yaml. -openstack_cloud: airstack - -# Ubuntu-24.04-Desktop (confirmed available on airlab-cloud). -image_id: 2ebb9061-8995-4238-a3cc-e230a3e863aa - -# OpenStack flavor with GPU + enough disk for Docker + sim images. -# Look up with: openstack flavor list -flavor_name: "gpu.rtxpro5000.1" - -# OpenStack network the ephemeral instance attaches to. Must allow outbound -# 443 to api.github.com (no inbound is required: the runner makes an outbound -# long-poll connection to GitHub). -network_name: "airstack.AirLab.Apps_group_network_gates" - -# OpenStack keypair injected into the instance for break-glass SSH access. -# The orchestrator never SSHes into workers itself. -keypair_name: "airstack-ci-cd" - -# Security group applied to spawned instances. Outbound 443 must be allowed. -security_group: "default" - -# OpenStack availability zone to spawn instances in (e.g. nova, gpu-zone-1). -# Leave empty to let Nova pick. -availability_zone: "gates" - -# If the chosen flavor has disk=0 (common for GPU flavors), Nova rejects -# direct image-boot with "Block Device Mapping is Invalid: You specified more -# local devices than the limit allows". Set this to >0 to boot from a Cinder -# volume of that size sourced from image_id (deleted on termination). Leave -# at 0 to boot directly from the image (only works for non-zero-disk flavors). -boot_volume_size_gb: 300 - -# Pre-allocated pool of floating IPs to rotate through for SSH access to -# workers. The orchestrator picks the first free IP from this list, in order, -# for each new spawn. When the worker is destroyed the IP auto-disassociates -# and returns to the pool. If non-empty, max_concurrent is capped at len(pool) -# so the orchestrator never spawns a worker it can't address. -# Allocate via: openstack floating ip create -# Leave empty to skip floating-IP attachment entirely. -floating_ips: [] -# Example: -# floating_ips: -# - 172.19.220.131 -# - 172.19.220.171 -# - 172.19.220.89 +# --- OSMO target --- + +# Path to the `osmo` CLI. The install script (see setup.sh / README) puts it on +# PATH as `osmo`; override with a full path if needed. +osmo_bin: "osmo" + +# URL of your OSMO web service (the control plane the CLI logs into). +osmo_url: "https://osmo.example.com" + +# File containing the OSMO service-account access token. This is the shared, +# non-personal "lab" identity — the analog of the old OpenStack application +# credential. The orchestrator runs `osmo login --method token --token-file` +# with it. Created by an OSMO admin via `osmo user create` + `osmo token set` +# (see README). It never leaves this host. +osmo_token_file: "/etc/airstack-orchestrator/osmo-token" + +# Dedicated CI GPU pool. Give this pool its own allocation so CI runs don't +# compete with researchers' interactive quotas. The service-account's role must +# grant workflow:Create/Cancel/Query scoped to this pool (pool/). +pool: "airstack-ci" + +# Optional platform (hardware type) to target within the pool. Leave empty to +# use the pool's default platform. List options with `osmo pool list` / the UI. +platform: "" + +# Scheduling priority: HIGH | NORMAL | LOW. +priority: "NORMAL" + +# --- Runner task (the per-job worker) --- + +# Prebaked image: Docker CE + compose + nvidia-container-toolkit + GH Actions +# runner. Build & push runner.Dockerfile to a registry the pool can pull from. +runner_image: "/airstack-ci-runner:2.334.0" + +# Resource request for the runner container. Size for the full stack build + +# sim (Isaac Sim / ms-airsim + robot + gcs) running under docker compose. +cpu: 8 +gpu: 1 +memory: "32Gi" +storage: "300Gi" + +# REQUIRED for the AirStack tests: they run `airstack up` (docker compose) +# inside the pod, which needs an inner Docker daemon -> a privileged container. +# The pool's platform must have "Privileged Mode Allowed" enabled by your OSMO +# admin, otherwise submission/scheduling will be rejected. +privileged: true + +# Use the node's host network for the runner container. Usually not needed +# (the runner only makes outbound calls to GitHub); leave false unless the +# tests need host networking. +host_network: false + +# GitHub Actions runner version to bake into runner_image. Kept here for +# reference/traceability; it is a build arg of runner.Dockerfile, not consumed +# by the orchestrator at runtime. Must match a tag at +# https://github.com/actions/runner/releases +runner_version: "2.334.0" # --- GitHub --- @@ -56,23 +65,22 @@ floating_ips: [] repo: "castacks/AirStack" # Labels the orchestrator polls for. A queued workflow_job whose `labels` -# array is a superset of this list gets a server spawned for it. +# array is a superset of this list gets a workflow submitted for it. These are +# unchanged from the OpenStack backend, so system-tests.yml needs no edits. runner_labels: - self-hosted - airstack-ephemeral -# GitHub Actions runner version (must exist as a release tag at -# https://github.com/actions/runner/releases). -runner_version: "2.334.0" - # --- Limits --- -# Maximum simultaneous in-flight ephemeral instances. +# Maximum simultaneous in-flight workflows the orchestrator will submit. OSMO +# queues anything beyond the pool's capacity on its own, but this caps how many +# jobs we hand it at once. max_concurrent: 3 -# Hard ceiling for a single job. Past this age the reaper force-deletes the -# server even if GitHub still reports the job as in-progress. Must comfortably -# exceed the longest expected job (autonomy/liveliness runs). +# Hard ceiling for a single job. Past this age the reaper cancels the workflow +# even if GitHub still reports the job as in-progress. Must comfortably exceed +# the longest expected job (liveliness / autonomy runs). max_job_minutes: 2880 # 48 hours # --- Polling intervals (seconds) --- @@ -80,8 +88,10 @@ max_job_minutes: 2880 # 48 hours spawn_poll_interval_s: 15 reap_poll_interval_s: 30 -# How long to wait for a freshly-created server to reach ACTIVE before -# treating the spawn as failed. If Nova flips the server to ERROR within this -# window the orchestrator logs the full fault (code/message/details/host/AZ) -# and deletes the server so the next iteration can retry cleanly. -server_active_timeout_s: 300 +# How long to wait for `osmo workflow submit` to return before treating the +# submission as failed. +submit_timeout_s: 180 + +# Name prefix for submitted workflows. Also used by the orphan sweep to find +# workflows this orchestrator owns. Keep the trailing dash. +workflow_name_prefix: "gha-runner-" diff --git a/.github/orchestrator/orchestrator.py b/.github/orchestrator/orchestrator.py index 3e65e906f..5af8ea540 100644 --- a/.github/orchestrator/orchestrator.py +++ b/.github/orchestrator/orchestrator.py @@ -1,56 +1,64 @@ #!/usr/bin/env python3 -"""AirStack CI orchestrator. +"""AirStack CI orchestrator (OSMO backend). Polls the GitHub API for queued workflow_jobs whose labels match this -orchestrator's runner_labels, and spawns truly ephemeral OpenStack instances -to execute them. Each ephemeral instance receives a single-use GitHub JIT -runner config via cloud-init; the GitHub PAT never leaves this orchestrator. +orchestrator's runner_labels, and submits truly ephemeral OSMO workflows to +execute them. Each workflow runs a single-job GitHub Actions runner in a +privileged, GPU-enabled container on an OSMO compute pool; the GitHub PAT never +leaves this orchestrator, and an OSMO service-account token (not a personal +account) is used only to submit / query / cancel workflows. + +This is a drop-in replacement for the previous OpenStack-Nova backend: the +GitHub side is unchanged (`runs-on: [self-hosted, airstack-ephemeral]`, the +single-use JIT runner config, the same-repo fork guard). Only the *spawn* +target changed from "create a Nova VM" to "submit an OSMO workflow". The +one-job-per-worker, destroy-after semantics are preserved — when the runner's +`run.sh` exits after a single job, the OSMO task completes and the pod is torn +down. Two cooperating loops: - - spawn loop: discover queued jobs, spawn one Nova server per job - - reap loop: delete servers whose jobs have completed, plus stragglers + - spawn loop: discover queued jobs, submit one OSMO workflow per job + - reap loop: cancel workflows whose jobs have completed, plus stragglers older than max_job_minutes and orphans not in state.json State persists in /var/lib/airstack-orchestrator/state.json so the -orchestrator can survive restarts without leaking instances. +orchestrator can survive restarts without leaking workflows. """ from __future__ import annotations import argparse -import base64 import json import logging import os +import re import signal +import subprocess import sys +import tempfile import threading import time from datetime import datetime, timezone from pathlib import Path from typing import Any -import openstack import requests import yaml from jinja2 import Template DEFAULT_CONFIG_PATH = "/etc/airstack-orchestrator/config.yaml" DEFAULT_PAT_PATH = "/etc/airstack-orchestrator/github-pat" +DEFAULT_OSMO_TOKEN_PATH = "/etc/airstack-orchestrator/osmo-token" DEFAULT_STATE_PATH = "/var/lib/airstack-orchestrator/state.json" -DEFAULT_TEMPLATE_PATH = "/opt/airstack-orchestrator/cloud-init.yaml.j2" - -# Metadata key/value applied to every Nova server we spawn. Used by the -# orphan reaper to identify servers we own even when state.json is missing. -ROLE_META_KEY = "airstack-role" -ROLE_META_VAL = "ephemeral-runner" -JOB_META_KEY = "airstack-job-id" +DEFAULT_TEMPLATE_PATH = "/opt/airstack-orchestrator/runner-workflow.yaml.j2" GITHUB_API = "https://api.github.com" log = logging.getLogger("orchestrator") +# ── file / state helpers ──────────────────────────────────────────────────── + def load_yaml(path: str) -> dict: with open(path) as f: return yaml.safe_load(f) @@ -76,6 +84,16 @@ def save_state(path: str, state: dict) -> None: os.replace(tmp, path) +def now_utc_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def parse_iso(s: str) -> datetime: + return datetime.fromisoformat(s) + + +# ── GitHub API (unchanged from the OpenStack backend) ───────────────────────── + def gh_request(method: str, path: str, pat: str, **kwargs: Any) -> Any: url = f"{GITHUB_API}{path}" headers = kwargs.pop("headers", {}) @@ -156,267 +174,129 @@ def get_job_status(repo: str, job_id: str, pat: str) -> dict | None: return r.json() -def render_cloud_init(template_path: str, encoded_jit_config: str, - runner_version: str) -> str: - with open(template_path) as f: - tmpl = Template(f.read()) - return tmpl.render( - encoded_jit_config=encoded_jit_config, - runner_version=runner_version, - ) - - -def spawn_server( - conn: openstack.connection.Connection, - config: dict, - name: str, - job_id: str, - user_data: str, -) -> str: - flavor = conn.compute.find_flavor(config["flavor_name"], ignore_missing=False) - network = conn.network.find_network(config["network_name"], ignore_missing=False) - create_kwargs = dict( - name=name, - flavor_id=flavor.id, - networks=[{"uuid": network.id}], - key_name=config["keypair_name"], - security_groups=[{"name": config["security_group"]}], - user_data=base64.b64encode(user_data.encode()).decode(), - metadata={ - ROLE_META_KEY: ROLE_META_VAL, - JOB_META_KEY: job_id, - }, - ) - - # Flavors with disk=0 (typical for GPU flavors on this cloud) cannot boot - # directly from an image — Nova rejects with "Block Device Mapping is - # Invalid: You specified more local devices than the limit allows". When - # boot_volume_size_gb is set, boot from a Cinder volume sourced from the - # image and delete it on termination. Otherwise fall back to direct image - # boot (works only when the flavor has a non-zero root disk). - boot_volume_size_gb = int(config.get("boot_volume_size_gb") or 0) - if boot_volume_size_gb > 0: - create_kwargs["block_device_mapping"] = [ - { - "uuid": config["image_id"], - "source_type": "image", - "destination_type": "volume", - "boot_index": 0, - "volume_size": boot_volume_size_gb, - "delete_on_termination": True, - } - ] - else: - create_kwargs["image_id"] = config["image_id"] - - az = config.get("availability_zone") - if az: - create_kwargs["availability_zone"] = az - server = conn.compute.create_server(**create_kwargs) - return server.id - - -def delete_server(conn: openstack.connection.Connection, server_id: str) -> None: +# ── OSMO CLI output parsing ─────────────────────────────────────────────────── +# +# The exact JSON keys returned by `osmo workflow {submit,query,list}` can vary +# slightly by OSMO release, so these parsers try a set of likely keys and fall +# back to scraping the human-readable text output. Verify the keys against your +# deployed version with `osmo workflow submit --dry-run` / `--format-type json` +# once and simplify if desired. + +_WF_ID_KEYS = ("workflow_id", "workflowId", "id", "uuid", "name", "workflow") +_STATUS_KEYS = ("status", "state", "workflow_status", "phase") +_KNOWN_STATUSES = { + "RUNNING", "PENDING", "WAITING", "COMPLETED", "FAILED", + "FAILED_EXEC_TIMEOUT", "FAILED_SERVER_ERROR", "FAILED_QUEUE_TIMEOUT", + "FAILED_SUBMISSION", "FAILED_CANCELED", "FAILED_BACKEND_ERROR", + "FAILED_IMAGE_PULL", "FAILED_EVICTED", "FAILED_START_ERROR", + "FAILED_START_TIMEOUT", "FAILED_PREEMPTED", +} +# Non-terminal statuses the orphan sweep considers "still alive". +_ACTIVE_STATUSES = ("RUNNING", "PENDING", "WAITING") + + +def _loads_or_none(text: str | None) -> Any: try: - conn.compute.delete_server(server_id, ignore_missing=True, force=True) - except Exception as e: - log.warning("delete_server(%s) failed: %s", server_id, e) - - -def list_owned_servers(conn: openstack.connection.Connection) -> list[Any]: - """List all Nova servers that carry our role metadata.""" - owned = [] - for s in conn.compute.servers(details=True): - meta = getattr(s, "metadata", None) or {} - if meta.get(ROLE_META_KEY) == ROLE_META_VAL: - owned.append(s) - return owned - - -def find_free_floating_ip( - conn: openstack.connection.Connection, pool: list[str] -) -> Any: - """Return the FloatingIP resource for the first address in `pool` that is - not currently associated with any port. Returns None if all are in use. - - Iterates `pool` in order so attachments rotate through it sequentially. - Logs a warning for any pool member that doesn't exist in this project. - """ - if not pool: + return json.loads(text) # type: ignore[arg-type] + except (json.JSONDecodeError, TypeError): return None - pool_set = set(pool) - fips_by_addr: dict[str, Any] = {} - for fip in conn.network.ips(): - if fip.floating_ip_address in pool_set: - fips_by_addr[fip.floating_ip_address] = fip - missing = pool_set - fips_by_addr.keys() - if missing: - log.warning( - "floating_ips configured but not found in this project: %s", - sorted(missing), - ) - for addr in pool: - fip = fips_by_addr.get(addr) - if fip is not None and not fip.port_id: - return fip - return None - -def check_flavor_capacity( - conn: openstack.connection.Connection, - flavor_name: str, -) -> tuple[bool, str]: - """Pre-flight: ask Nova's placement API whether any host can satisfy this - flavor's resource request right now. - Returns (ok, reason). When ok=False the orchestrator should defer the - spawn iteration; reason is a one-line human-readable explanation - (e.g. "no host can satisfy {'VCPU': 8, 'MEMORY_MB': 32768, 'VGPU': 1}"). - - If the placement API can't be queried for any reason we return - (True, "") and let Nova make the call. The - pre-flight is a fast-path optimization, not a gate — Nova still has the - final say at create_server time (and ERROR-status fallback handles - anything we miss). - """ - try: - flavor = conn.compute.find_flavor(flavor_name, ignore_missing=False) - except Exception as e: - return True, f"flavor lookup failed: {e}" - - # Standard resources every Nova flavor expresses. - resources: dict[str, int] = {} - if getattr(flavor, "vcpus", 0): - resources["VCPU"] = int(flavor.vcpus) - if getattr(flavor, "ram", 0): - resources["MEMORY_MB"] = int(flavor.ram) - if getattr(flavor, "disk", 0): - resources["DISK_GB"] = int(flavor.disk) - - # Custom / specialized resources (VGPU, PCI_*, CUSTOM_*) come from the - # flavor's extra_specs as `resources:=`. This is how Nova - # itself learns to ask placement for GPU capacity. - extra = getattr(flavor, "extra_specs", {}) or {} - for k, v in extra.items(): - if not k.startswith("resources:"): - continue - rc = k.split(":", 1)[1] - try: - resources[rc] = int(v) - except (TypeError, ValueError): - pass - - if not resources: - return True, "flavor expresses no resources — skipping placement check" +def _first_str(d: dict, keys: tuple[str, ...]) -> str | None: + for k in keys: + v = d.get(k) + if isinstance(v, str) and v: + return v + return None - try: - result = conn.placement.allocation_candidates( - resources=resources, limit=1, - ) - if hasattr(result, "allocation_requests"): - candidates = list(result.allocation_requests or []) - else: - candidates = list(result) - except Exception as e: - return True, f"placement query failed ({type(e).__name__}: {e})" - - if candidates: - return True, "" - return False, f"no host can satisfy {resources}" - - -def wait_for_server_active( - conn: openstack.connection.Connection, - server_id: str, - timeout_s: int = 300, - poll_interval_s: float = 3.0, -) -> Any: - """Poll Nova until the server is ACTIVE. Raise with full context if it - enters ERROR or never reaches ACTIVE in time. - - Nova surfaces the actual reason for an ERROR via the `fault` attribute - (message + code + details), so we log it verbatim. We also include - task_state / vm_state / power_state because Nova sometimes leaves the - fault empty and these tell you whether the failure was at scheduling, - networking, or block-device-mapping time. - """ - deadline = time.monotonic() + timeout_s - last_status = "?" - last_task = None - while time.monotonic() < deadline: - s = conn.compute.get_server(server_id) - status = getattr(s, "status", "UNKNOWN") or "UNKNOWN" - task = ( - getattr(s, "task_state", None) - or getattr(s, "OS-EXT-STS:task_state", None) - ) - if status != last_status or task != last_task: - log.info( - "server %s status=%s task_state=%s", server_id, status, task, - ) - last_status, last_task = status, task - if status == "ACTIVE": +def _extract_workflow_id(stdout: str | None) -> str | None: + data = _loads_or_none(stdout) + if isinstance(data, dict): + wid = _first_str(data, _WF_ID_KEYS) + if wid: + return wid + wf = data.get("workflow") + if isinstance(wf, dict): + wid = _first_str(wf, _WF_ID_KEYS) + if wid: + return wid + m = re.search(r"Workflow\s*ID\s*[-:]\s*(\S+)", stdout or "", re.IGNORECASE) + return m.group(1) if m else None + + +def _extract_status(stdout: str | None) -> str | None: + data = _loads_or_none(stdout) + if isinstance(data, dict): + st = _first_str(data, _STATUS_KEYS) + if st: + return st.upper() + wf = data.get("workflow") + if isinstance(wf, dict): + st = _first_str(wf, _STATUS_KEYS) + if st: + return st.upper() + up = (stdout or "").upper() + for s in sorted(_KNOWN_STATUSES, key=len, reverse=True): + if s in up: return s + return None - if status == "ERROR": - fault = getattr(s, "fault", None) or {} - vm_state = ( - getattr(s, "vm_state", None) - or getattr(s, "OS-EXT-STS:vm_state", None) - ) - power_state = ( - getattr(s, "power_state", None) - or getattr(s, "OS-EXT-STS:power_state", None) - ) - host = getattr(s, "compute_host", None) or getattr( - s, "OS-EXT-SRV-ATTR:host", None - ) - az = getattr(s, "availability_zone", None) or getattr( - s, "OS-EXT-AZ:availability_zone", None - ) - raise RuntimeError( - "server " - + str(server_id) - + " entered ERROR: " - + f"fault.code={fault.get('code')!r} " - + f"fault.message={fault.get('message')!r} " - + f"fault.details={fault.get('details')!r} " - + f"task_state={task!r} vm_state={vm_state!r} " - + f"power_state={power_state!r} host={host!r} az={az!r}" - ) - time.sleep(poll_interval_s) +def _extract_workflow_list(stdout: str | None) -> list[dict]: + data = _loads_or_none(stdout) + if isinstance(data, dict): + for key in ("workflows", "items", "results", "data"): + if isinstance(data.get(key), list): + data = data[key] + break + items: list[dict] = [] + if isinstance(data, list): + for entry in data: + if not isinstance(entry, dict): + continue + wid = _first_str(entry, _WF_ID_KEYS) + name = entry.get("name") if isinstance(entry.get("name"), str) else None + status = _first_str(entry, _STATUS_KEYS) + if wid or name: + items.append( + {"id": wid, "name": name, + "status": status.upper() if status else None} + ) + return items + - raise RuntimeError( - f"server {server_id} did not reach ACTIVE within {timeout_s}s " - f"(last status={last_status!r} task_state={last_task!r})" - ) +def _is_terminal(status: str | None) -> bool: + if not status: + return False + return status == "COMPLETED" or status.startswith("FAILED") -def attach_floating_ip( - conn: openstack.connection.Connection, server_id: str, fip: Any -) -> str: - """Wait for the server to have a network port, then associate `fip`. - Returns the floating IP address.""" - for _ in range(60): # ~120s - ports = list(conn.network.ports(device_id=server_id)) - if ports: - break - time.sleep(2) - else: - raise RuntimeError(f"server {server_id} got no network port within 120s") - conn.network.update_ip(fip, port_id=ports[0].id) - return fip.floating_ip_address +def _looks_like_auth_error(r: subprocess.CompletedProcess) -> bool: + blob = f"{r.stdout or ''}\n{r.stderr or ''}".lower() + markers = ("401", "403", "unauthorized", "forbidden", "expired", + "not logged in", "please login", "authentication", + "invalid token", "token is invalid") + return any(m in blob for m in markers) -def now_utc_iso() -> str: - return datetime.now(timezone.utc).isoformat() +def _name_age_minutes(name: str | None) -> float | None: + """Age in minutes parsed from our `...-` name suffix, or None. + OSMO may append its own suffix after the name we submit, so we match the + first 10+ digit run (the unix timestamp) even when trailing chars follow. + """ + m = re.search(r"-(\d{10,})(?:\D.*)?$", name or "") + if not m: + return None + try: + ts = int(m.group(1)) + except ValueError: + return None + return (time.time() - ts) / 60.0 -def parse_iso(s: str) -> datetime: - return datetime.fromisoformat(s) +# ── orchestrator ────────────────────────────────────────────────────────────── class Orchestrator: def __init__(self, config: dict, pat: str, state_path: str, template_path: str): @@ -424,230 +304,286 @@ def __init__(self, config: dict, pat: str, state_path: str, template_path: str): self.pat = pat self.state_path = state_path self.template_path = template_path - self.conn = openstack.connect(cloud=config.get("openstack_cloud", "airstack")) + + # OSMO target. + self.osmo_bin = config.get("osmo_bin", "osmo") + self.osmo_url = config["osmo_url"] + self.token_file = config.get("osmo_token_file", DEFAULT_OSMO_TOKEN_PATH) + self.pool = config["pool"] + self.platform = config.get("platform", "") or "" + self.priority = str(config.get("priority", "NORMAL")).upper() + + # Runner task shape. + self.runner_image = config["runner_image"] + self.cpu = config.get("cpu", 8) + self.gpu = config.get("gpu", 1) + self.memory = config.get("memory", "32Gi") + self.storage = config.get("storage", "300Gi") + self.privileged = bool(config.get("privileged", True)) + self.host_network = bool(config.get("host_network", False)) + + # GitHub. self.repo = config["repo"] self.runner_labels = config["runner_labels"] - self.runner_version = config["runner_version"] + + # Limits / timing. self.max_concurrent = int(config.get("max_concurrent", 3)) - self.floating_ips: list[str] = list(config.get("floating_ips") or []) - # Cap spawns to FIP pool size so we never queue jobs we can't address. - self.effective_max_concurrent = self.max_concurrent - if self.floating_ips: - self.effective_max_concurrent = min( - self.max_concurrent, len(self.floating_ips) - ) - self.max_job_minutes = int(config.get("max_job_minutes", 90)) + self.max_job_minutes = int(config.get("max_job_minutes", 2880)) self.spawn_interval = int(config.get("spawn_poll_interval_s", 15)) self.reap_interval = int(config.get("reap_poll_interval_s", 30)) + self.submit_timeout = int(config.get("submit_timeout_s", 180)) + self.workflow_prefix = config.get("workflow_name_prefix", "gha-runner-") + self.stop_evt = threading.Event() + # Establish the OSMO session up-front for early feedback; individual + # commands re-login on demand if the session lapses. + self._login() + def stop(self, *_: Any) -> None: log.info("stop signal received; draining loops") self.stop_evt.set() + # ── OSMO CLI plumbing ──────────────────────────────────────────────────── + + def _run_osmo(self, args: list[str], timeout: int) -> subprocess.CompletedProcess: + return subprocess.run( + [self.osmo_bin, *args], + capture_output=True, text=True, timeout=timeout, + ) + + def _login(self) -> bool: + try: + r = self._run_osmo( + ["login", self.osmo_url, "--method", "token", + "--token-file", self.token_file], + timeout=60, + ) + except Exception as e: # noqa: BLE001 - startup best-effort + log.warning("osmo login raised: %s", e) + return False + if r.returncode != 0: + log.warning( + "osmo login failed (rc=%d): %s", + r.returncode, (r.stderr or r.stdout).strip(), + ) + return False + log.info("osmo login succeeded (url=%s, token_file=%s)", + self.osmo_url, self.token_file) + return True + + def _osmo(self, args: list[str], timeout: int, + relogin: bool = True) -> subprocess.CompletedProcess: + """Run an osmo CLI command, re-logging-in once on an auth failure.""" + r = self._run_osmo(args, timeout=timeout) + if r.returncode != 0 and relogin and _looks_like_auth_error(r): + log.info("osmo command hit an auth error; re-logging in and retrying") + if self._login(): + r = self._run_osmo(args, timeout=timeout) + return r + + def submit_workflow(self, workflow_file: str) -> str: + args = ["workflow", "submit", workflow_file, "--pool", self.pool, + "--priority", self.priority, "--format-type", "json"] + r = self._osmo(args, timeout=self.submit_timeout) + if r.returncode != 0: + raise RuntimeError( + f"osmo workflow submit failed (rc={r.returncode}): " + f"{(r.stderr or r.stdout).strip()}" + ) + wid = _extract_workflow_id(r.stdout) or _extract_workflow_id(r.stderr) + if not wid: + raise RuntimeError( + "could not parse workflow id from submit output: " + f"{(r.stdout or '').strip()[:500]}" + ) + return wid + + def query_status(self, workflow_id: str) -> str | None: + r = self._osmo( + ["workflow", "query", workflow_id, "--format-type", "json"], + timeout=60, + ) + if r.returncode != 0: + log.debug("osmo workflow query %s failed: %s", + workflow_id, (r.stderr or r.stdout).strip()) + return None + return _extract_status(r.stdout) or _extract_status(r.stderr) + + def cancel_workflow(self, workflow_id: str) -> None: + r = self._osmo( + ["workflow", "cancel", workflow_id, "--force", + "--message", "orchestrator reap", "--format-type", "json"], + timeout=60, + ) + if r.returncode != 0: + log.warning("osmo workflow cancel %s failed (rc=%d): %s", + workflow_id, r.returncode, (r.stderr or r.stdout).strip()) + + def list_runner_workflows(self) -> list[dict]: + """Active workflows (RUNNING/PENDING/WAITING) named with our prefix.""" + r = self._osmo( + ["workflow", "list", "--name", self.workflow_prefix, + "--pool", self.pool, "--count", "100", + "--status", *_ACTIVE_STATUSES, "--format-type", "json"], + timeout=60, + ) + if r.returncode != 0: + log.warning("osmo workflow list failed: %s", + (r.stderr or r.stdout).strip()) + return [] + return _extract_workflow_list(r.stdout) + + # ── workflow rendering ─────────────────────────────────────────────────── + + def render_workflow(self, workflow_name: str, encoded_jit_config: str) -> str: + with open(self.template_path) as f: + tmpl = Template(f.read()) + return tmpl.render( + workflow_name=workflow_name, + runner_image=self.runner_image, + cpu=self.cpu, + gpu=self.gpu, + memory=self.memory, + storage=self.storage, + platform=self.platform, + privileged="true" if self.privileged else "false", + host_network="true" if self.host_network else "false", + encoded_jit_config=encoded_jit_config, + runner_labels=self.runner_labels, + ) + + def _write_temp_workflow(self, name: str, content: str) -> str: + fd, path = tempfile.mkstemp(prefix=f"{name}-", suffix=".yaml") + with os.fdopen(fd, "w") as f: + f.write(content) + return path + + # ── loops ──────────────────────────────────────────────────────────────── + def spawn_once(self) -> None: state = load_state(self.state_path) active = len(state["jobs"]) - if active >= self.effective_max_concurrent: + if active >= self.max_concurrent: return try: queued = find_queued_jobs(self.repo, self.runner_labels, self.pat) - except Exception as e: + except Exception as e: # noqa: BLE001 log.warning("find_queued_jobs failed: %s", e) return - # Pre-flight capacity check via placement API. Every queued job uses - # the same flavor, so we check once per iteration. When OpenStack is - # out of GPUs / vCPU / RAM we defer the whole iteration — better than - # burning JIT tokens on creates that Nova will flip to ERROR. The - # next iteration retries automatically. - if queued: - ok, reason = check_flavor_capacity( - self.conn, self.config["flavor_name"] - ) - if not ok: - log.warning( - "deferring spawn — OpenStack capacity unavailable: %s. " - "Will retry in %ds.", - reason, self.spawn_interval, - ) - return - elif reason: - # Soft-skip path: placement check couldn't run (e.g. older - # Nova). Surface why so it's debuggable, then proceed. - log.debug("placement pre-flight: %s", reason) - for job in queued: - if active >= self.effective_max_concurrent: + if active >= self.max_concurrent: break job_id = job["job_id"] if job_id in state["jobs"]: continue - # Pre-check FIP availability before minting a JIT token so we - # don't burn one when there's nowhere to attach the worker. - reserved_fip = None - if self.floating_ips: - reserved_fip = find_free_floating_ip(self.conn, self.floating_ips) - if reserved_fip is None: - log.warning( - "no free floating IP in pool (%d configured); " - "deferring spawns until one frees up", - len(self.floating_ips), - ) - break - ts = int(time.time()) - runner_name = f"ephemeral-{job_id}-{ts}" - server_id: str | None = None + # OSMO workflow name doubles as the JIT runner registration name. + workflow_name = f"{self.workflow_prefix}{job_id}-{ts}" + tmp_path: str | None = None try: jit = mint_jit_config( - self.repo, runner_name, self.runner_labels, self.pat + self.repo, workflow_name, self.runner_labels, self.pat ) - user_data = render_cloud_init( - self.template_path, jit, self.runner_version - ) - server_id = spawn_server( - self.conn, self.config, runner_name, job_id, user_data - ) - # Don't move on until Nova reports ACTIVE. If it transitions - # to ERROR, this raises with the Nova fault details so the - # operator can see *why* the spawn failed (quota, scheduling, - # block-device-mapping, networking, etc.). - wait_for_server_active( - self.conn, - server_id, - timeout_s=int(self.config.get("server_active_timeout_s", 300)), - ) - except Exception as e: - # Tag capacity-related Nova faults so log-grepping for - # "capacity unavailable" finds both the pre-flight defer and - # the post-create fallback (e.g. PCI passthrough that - # placement doesn't track). - msg = str(e).lower() - capacity_markers = ( - "no valid host", - "insufficient", - "quotaexceeded", - "out of resource", - "no host can satisfy", - "no allocation candidates", - ) - if any(m in msg for m in capacity_markers): - log.warning( - "spawn failed for job %s — OpenStack capacity " - "unavailable (post-create): %s. Will retry in %ds.", - job_id, e, self.spawn_interval, - ) - else: - log.exception("spawn failed for job %s: %s", job_id, e) - if server_id: - log.warning( - "deleting failed server %s to release its volume / FIP", - server_id, - ) - delete_server(self.conn, server_id) + workflow_yaml = self.render_workflow(workflow_name, jit) + tmp_path = self._write_temp_workflow(workflow_name, workflow_yaml) + workflow_id = self.submit_workflow(tmp_path) + except Exception as e: # noqa: BLE001 + log.exception("submit failed for job %s: %s", job_id, e) continue - - floating_ip_addr: str | None = None - if reserved_fip is not None: - try: - floating_ip_addr = attach_floating_ip( - self.conn, server_id, reserved_fip - ) - log.info( - "attached floating IP %s to server %s (job %s)", - floating_ip_addr, server_id, job_id, - ) - except Exception as e: - log.exception( - "FIP attach failed for server %s; deleting to avoid " - "leaking a worker without external access: %s", - server_id, e, - ) - delete_server(self.conn, server_id) - continue + finally: + if tmp_path: + try: + os.remove(tmp_path) + except OSError: + pass state["jobs"][job_id] = { "run_id": job["run_id"], - "server_id": server_id, - "runner_name": runner_name, - "spawned_at": now_utc_iso(), + "workflow_id": workflow_id, + "workflow_name": workflow_name, + "runner_name": workflow_name, + "submitted_at": now_utc_iso(), "name": job["name"], - "floating_ip": floating_ip_addr, } save_state(self.state_path, state) active += 1 log.info( - "spawned server %s for job %s (%s)", server_id, job_id, job["name"] + "submitted workflow %s for job %s (%s)", + workflow_id, job_id, job["name"], ) def reap_once(self) -> None: state = load_state(self.state_path) now = datetime.now(timezone.utc) - # 1. Delete servers for completed jobs. + # 1. Cancel workflows for completed / purged jobs. for job_id in list(state["jobs"].keys()): entry = state["jobs"][job_id] + wid = entry["workflow_id"] try: job = get_job_status(self.repo, job_id, self.pat) - except Exception as e: + except Exception as e: # noqa: BLE001 log.warning("get_job_status(%s) failed: %s", job_id, e) continue + if job is None or job.get("status") == "completed": - log.info("reaping server %s (job %s done)", entry["server_id"], job_id) - delete_server(self.conn, entry["server_id"]) + # The runner usually exits on its own (task self-completes and + # the pod is torn down); only cancel if it's somehow still live. + status = self.query_status(wid) + if not _is_terminal(status): + log.info("reaping workflow %s (job %s done, wf status=%s)", + wid, job_id, status) + self.cancel_workflow(wid) + else: + log.info("workflow %s already terminal (%s) for job %s", + wid, status, job_id) del state["jobs"][job_id] continue # 2. Force-reap stragglers older than max_job_minutes. - spawned = parse_iso(entry["spawned_at"]) - age_min = (now - spawned).total_seconds() / 60.0 + age_min = (now - parse_iso(entry["submitted_at"])).total_seconds() / 60.0 if age_min > self.max_job_minutes: log.warning( - "force-reaping server %s (job %s age %.1fm > %dm)", - entry["server_id"], job_id, age_min, self.max_job_minutes, + "force-reaping workflow %s (job %s age %.1fm > %dm)", + wid, job_id, age_min, self.max_job_minutes, ) - delete_server(self.conn, entry["server_id"]) + self.cancel_workflow(wid) del state["jobs"][job_id] save_state(self.state_path, state) - # 3. Orphan sweep: any server we own that isn't in state and isn't - # in the brief just-spawned window. Catches state.json wipes and - # crashes between spawn and save_state. + # 3. Orphan sweep: our-named workflows still active but absent from + # state (catches state.json wipes and crashes between submit and + # save_state). Skip very fresh ones so we don't race our own submit. try: - owned = list_owned_servers(self.conn) - except Exception as e: - log.warning("list_owned_servers failed: %s", e) + listed = self.list_runner_workflows() + except Exception as e: # noqa: BLE001 + log.warning("list_runner_workflows failed: %s", e) return - tracked_ids = {e["server_id"] for e in state["jobs"].values()} - for s in owned: - if s.id in tracked_ids: + tracked_ids = {e["workflow_id"] for e in state["jobs"].values()} + tracked_names = {e["workflow_name"] for e in state["jobs"].values()} + for wf in listed: + wid, wname = wf.get("id"), wf.get("name") + if (wid and wid in tracked_ids) or (wname and wname in tracked_names): continue - created = getattr(s, "created_at", None) - if created: - try: - age_min = (now - parse_iso(created.replace("Z", "+00:00"))).total_seconds() / 60.0 - except Exception: - age_min = self.max_job_minutes + 1 - else: - age_min = self.max_job_minutes + 1 - # Only reap orphans that have lived past one spawn interval - # (to avoid racing our own freshly-created server). - if age_min < 2: + age = _name_age_minutes(wname) + if age is not None and age < 2: continue - log.warning( - "orphan-reaping server %s (not in state, age %.1fm)", s.id, age_min - ) - delete_server(self.conn, s.id) + target = wid or wname + if not target: + continue + log.warning("orphan-reaping workflow %s (not in state)", target) + self.cancel_workflow(target) def run(self) -> None: log.info( - "orchestrator started: repo=%s labels=%s max_concurrent=%d " - "(effective=%d, floating_ip_pool=%d)", - self.repo, self.runner_labels, self.max_concurrent, - self.effective_max_concurrent, len(self.floating_ips), + "orchestrator started (OSMO backend): repo=%s labels=%s pool=%s " + "platform=%s max_concurrent=%d", + self.repo, self.runner_labels, self.pool, + self.platform or "(pool default)", self.max_concurrent, ) last_spawn = 0.0 last_reap = 0.0 @@ -656,13 +592,13 @@ def run(self) -> None: if now - last_spawn >= self.spawn_interval: try: self.spawn_once() - except Exception: + except Exception: # noqa: BLE001 log.exception("spawn loop iteration failed") last_spawn = now if now - last_reap >= self.reap_interval: try: self.reap_once() - except Exception: + except Exception: # noqa: BLE001 log.exception("reap loop iteration failed") last_reap = now self.stop_evt.wait(timeout=1.0) diff --git a/.github/orchestrator/requirements.txt b/.github/orchestrator/requirements.txt index b69702b59..f710f7167 100644 --- a/.github/orchestrator/requirements.txt +++ b/.github/orchestrator/requirements.txt @@ -1,4 +1,3 @@ -openstacksdk>=3.0,<5 requests>=2.31 PyYAML>=6.0 Jinja2>=3.1 diff --git a/.github/orchestrator/runner-entrypoint.sh b/.github/orchestrator/runner-entrypoint.sh new file mode 100644 index 000000000..cc5696ae8 --- /dev/null +++ b/.github/orchestrator/runner-entrypoint.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Entry point for the AirStack CI ephemeral-runner container (an OSMO task). +# +# Starts an inner Docker daemon — the AirStack test harness runs `airstack up` +# (docker compose) inside this container — waits for it, then runs exactly ONE +# ephemeral GitHub Actions job via the single-use JIT config. When run.sh exits, +# the OSMO task completes and the pod is destroyed: one job per pod, same as the +# old OpenStack VM. +# +# Requires a privileged pod (dockerd) scheduled on a GPU platform; the NVIDIA +# container toolkit (baked into the image) lets the inner dockerd pass the node +# GPU through to the compose containers. +set -euxo pipefail + +: "${ENCODED_JIT_CONFIG:?ENCODED_JIT_CONFIG must be set by the workflow}" + +# Start dockerd in the background (needs privileged). +dockerd >/var/log/dockerd.log 2>&1 & + +# Wait for the daemon to accept connections (~60s budget). +for _ in $(seq 1 60); do + if docker info >/dev/null 2>&1; then + break + fi + sleep 1 +done +if ! docker info >/dev/null 2>&1; then + echo "ERROR: dockerd did not become ready" >&2 + cat /var/log/dockerd.log >&2 || true + exit 1 +fi + +# Non-fatal GPU sanity check — surfaces GPU/privileged/toolkit misconfig early. +nvidia-smi || echo "WARN: nvidia-smi unavailable (check GPU + privileged + toolkit)" + +cd /home/runner/actions-runner +# The JIT config makes this runner single-use + ephemeral; run.sh returns after +# one job, which completes the task and lets OSMO reap the pod. +exec ./run.sh --jitconfig "${ENCODED_JIT_CONFIG}" diff --git a/.github/orchestrator/runner-workflow.yaml.j2 b/.github/orchestrator/runner-workflow.yaml.j2 new file mode 100644 index 000000000..124210ddb --- /dev/null +++ b/.github/orchestrator/runner-workflow.yaml.j2 @@ -0,0 +1,48 @@ +# OSMO workflow rendered per-job by orchestrator.py (replaces the old +# cloud-init.yaml.j2). One workflow == one ephemeral GitHub Actions runner == +# one CI job. Jinja variables injected by the orchestrator: +# +# workflow_name unique name (gha-runner--); also the +# JIT runner registration name +# runner_image prebaked image (docker-ce + compose + nvidia-container- +# toolkit + GH Actions runner) — see runner.Dockerfile +# cpu / gpu / memory / storage resource request for the runner task +# platform optional OSMO platform to target within the pool +# (omitted -> the pool's default platform) +# privileged "true"/"false"; MUST be "true" because the AirStack +# test harness runs `airstack up` (docker compose) inside +# the pod, which needs an inner Docker daemon. Requires a +# platform with "Privileged Mode Allowed" (ask your OSMO +# admin to enable it for the CI pool). +# host_network "true"/"false" +# encoded_jit_config single-use base64 GitHub JIT runner config +# +# The pool is passed by the orchestrator via `osmo workflow submit --pool`, so +# it is intentionally not hard-coded here. +# +# Lifecycle: the container starts dockerd, then runs exactly ONE ephemeral job +# via the JIT config. When run.sh exits, the task completes and OSMO tears the +# pod down — same "destroy after one job" behavior the OpenStack VM had. +workflow: + name: {{ workflow_name }} + resources: + runner: + cpu: {{ cpu }} + gpu: {{ gpu }} + memory: {{ memory }} + storage: {{ storage }} +{% if platform %} platform: {{ platform }} +{% endif %} + tasks: + - name: runner + image: {{ runner_image }} + resource: runner + privileged: {{ privileged }} +{% if host_network == "true" %} hostNetwork: true +{% endif %} + environment: + # Single-use + ephemeral: the runner exits after exactly one job. + ENCODED_JIT_CONFIG: "{{ encoded_jit_config }}" + # The runner refuses to run as root without this; the DinD image is root. + RUNNER_ALLOW_RUNASROOT: "1" + command: ["/usr/local/bin/run-ephemeral-runner.sh"] diff --git a/.github/orchestrator/runner.Dockerfile b/.github/orchestrator/runner.Dockerfile new file mode 100644 index 000000000..05af72566 --- /dev/null +++ b/.github/orchestrator/runner.Dockerfile @@ -0,0 +1,62 @@ +# Prebaked image for AirStack CI ephemeral runners on OSMO. +# +# This bakes in what the old cloud-init.yaml.j2 installed on the OpenStack VM +# (Docker CE + compose plugin, NVIDIA container toolkit, the GitHub Actions +# runner) so pod start is fast and the single-use JIT token can't expire during +# a slow apt/bootstrap. Build it and push to a registry your OSMO pool can pull: +# +# docker build -f runner.Dockerfile \ +# --build-arg RUNNER_VERSION=2.334.0 \ +# -t /airstack-ci-runner:2.334.0 . +# docker push /airstack-ci-runner:2.334.0 +# +# Then set `runner_image: /airstack-ci-runner:2.334.0` in config.yaml. +# Keep RUNNER_VERSION in sync with the actions/runner release you want. +# +# GPU-in-Docker-in-Docker: the OSMO task must run privileged (see +# `privileged: true` in runner-workflow.yaml.j2) on a platform with +# "Privileged Mode Allowed" + GPUs. The inner dockerd uses the NVIDIA container +# toolkit installed here to expose the node GPU to the `airstack up` containers. +# The image is linux/amd64 (x86_64 runner tarball); rebuild for arm64 if needed. +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# Docker CE (+ compose/buildx plugins), NVIDIA container toolkit, and the tools +# the AirStack test harness / GH runner need (git, jq, python venv, ...). +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl gnupg jq git sudo iproute2 \ + python3 python3-venv python3-pip \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ +https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + > /etc/apt/sources.list.d/docker.list \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ + | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ + > /etc/apt/sources.list.d/nvidia-container-toolkit.list \ + && apt-get update && apt-get install -y --no-install-recommends \ + docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin \ + nvidia-container-toolkit \ + && nvidia-ctk runtime configure --runtime=docker \ + && rm -rf /var/lib/apt/lists/* + +# GitHub Actions runner (self-contained; version pinned at build time). +ARG RUNNER_VERSION=2.334.0 +RUN mkdir -p /home/runner/actions-runner \ + && cd /home/runner/actions-runner \ + && curl -fsSL -o runner.tar.gz \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" \ + && tar xzf runner.tar.gz \ + && rm runner.tar.gz \ + && ./bin/installdependencies.sh + +COPY runner-entrypoint.sh /usr/local/bin/run-ephemeral-runner.sh +RUN chmod +x /usr/local/bin/run-ephemeral-runner.sh + +WORKDIR /home/runner/actions-runner diff --git a/.github/orchestrator/setup.sh b/.github/orchestrator/setup.sh index 4803b4a33..adb3c078f 100755 --- a/.github/orchestrator/setup.sh +++ b/.github/orchestrator/setup.sh @@ -1,13 +1,17 @@ #!/usr/bin/env bash -# One-time orchestrator-VM setup. Run as root on the airstack-ci-cd-orchestrator -# OpenStack instance after cloning the repo. +# One-time orchestrator-VM setup (OSMO backend). Run as root on the +# airstack-ci-cd-orchestrator instance after cloning the repo. # # Pre-reqs (do these *before* running this script): -# 1. ~/.config/openstack/clouds.yaml staged for the orchestrator user -# (application credential — see .github/orchestrator/README.md). -# 2. /tmp/github-pat exists with the GitHub PAT contents. +# 1. /tmp/github-pat exists with the GitHub PAT contents. +# 2. /tmp/osmo-token exists with the OSMO service-account access token +# (from `osmo token set` — see .github/orchestrator/README.md). Optional +# at setup time; you can stage it later before starting the service. # 3. This repo cloned somewhere readable (this script copies code out of # its containing directory). +# +# The orchestrator host is lightweight and needs NO GPU — it only polls GitHub +# and submits OSMO workflows. 1 vCPU / 2GB RAM / 20GB disk is plenty. set -euo pipefail @@ -30,18 +34,33 @@ fi echo "==> Installing system packages" apt-get update -apt-get install -y python3 python3-venv python3-pip +apt-get install -y python3 python3-venv python3-pip curl ca-certificates + +echo "==> Installing the OSMO CLI" +if command -v osmo >/dev/null 2>&1; then + echo " osmo already installed ($(command -v osmo)); skipping" +else + # Latest client. Pin to a release from https://github.com/NVIDIA/OSMO/releases + # if you need a specific version. + curl -fsSL https://raw.githubusercontent.com/NVIDIA/OSMO/refs/heads/main/install.sh | bash + command -v osmo >/dev/null 2>&1 \ + || echo "WARNING: osmo not on PATH after install — check the installer output" >&2 +fi echo "==> Creating directories" install -d -o "$USER_NAME" -g "$USER_NAME" -m 0750 "$INSTALL_DIR" install -d -o root -g "$USER_NAME" -m 0750 "$CONFIG_DIR" install -d -o "$USER_NAME" -g "$USER_NAME" -m 0750 "$STATE_DIR" +# XDG dirs for the osmo CLI login session (see the systemd unit's HOME/XDG env). +install -d -o "$USER_NAME" -g "$USER_NAME" -m 0700 "$STATE_DIR/.config" +install -d -o "$USER_NAME" -g "$USER_NAME" -m 0700 "$STATE_DIR/.cache" +install -d -o "$USER_NAME" -g "$USER_NAME" -m 0700 "$STATE_DIR/.state" echo "==> Copying orchestrator files to $INSTALL_DIR" install -o "$USER_NAME" -g "$USER_NAME" -m 0755 \ "$REPO_DIR/orchestrator.py" "$INSTALL_DIR/orchestrator.py" install -o "$USER_NAME" -g "$USER_NAME" -m 0644 \ - "$REPO_DIR/cloud-init.yaml.j2" "$INSTALL_DIR/cloud-init.yaml.j2" + "$REPO_DIR/runner-workflow.yaml.j2" "$INSTALL_DIR/runner-workflow.yaml.j2" echo "==> Building Python venv" sudo -u "$USER_NAME" python3 -m venv "$INSTALL_DIR/venv" @@ -63,11 +82,14 @@ fi install -o root -g "$USER_NAME" -m 0640 /tmp/github-pat "$CONFIG_DIR/github-pat" shred -u /tmp/github-pat -echo "==> Verifying clouds.yaml" -CLOUDS_YAML="/home/$USER_NAME/.config/openstack/clouds.yaml" -if [[ ! -f "$CLOUDS_YAML" ]]; then - echo "WARNING: $CLOUDS_YAML missing." >&2 - echo " Create it (application credential) before starting the service." >&2 +echo "==> Installing OSMO service-account token (from /tmp/osmo-token)" +if [[ -f /tmp/osmo-token ]]; then + install -o root -g "$USER_NAME" -m 0640 /tmp/osmo-token "$CONFIG_DIR/osmo-token" + shred -u /tmp/osmo-token +else + echo "WARNING: /tmp/osmo-token not found." >&2 + echo " Stage the OSMO service-account token before starting the service:" >&2 + echo " sudo install -o root -g $USER_NAME -m 0640 /tmp/osmo-token $CONFIG_DIR/osmo-token" >&2 fi echo "==> Installing systemd unit" @@ -78,7 +100,8 @@ systemctl daemon-reload echo echo "Setup complete. Next steps:" -echo " 1. Edit $CONFIG_DIR/config.yaml — fill flavor/network/keypair/security_group." -echo " 2. Verify $CLOUDS_YAML exists with the application credential." +echo " 1. Edit $CONFIG_DIR/config.yaml — set osmo_url, pool, platform," +echo " runner_image, and resources." +echo " 2. Ensure $CONFIG_DIR/osmo-token holds the OSMO service-account token." echo " 3. systemctl enable --now airstack-orchestrator.service" echo " 4. journalctl -u airstack-orchestrator.service -f" diff --git a/AGENTS.md b/AGENTS.md index 0a6b86015..4207c2b87 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ AirStack/ ├── tests/ # System tests (pytest) + metrics reporting ├── .github/ │ ├── workflows/ # GitHub Actions CI (system-tests, docker-build, etc.) -│ └── orchestrator/ # OpenStack-backed ephemeral self-hosted runners +│ └── orchestrator/ # OSMO-backed ephemeral self-hosted runners └── .agents/skills/ # Detailed workflow guides for agents ``` @@ -268,16 +268,16 @@ GitHub Actions workflows live in [`.github/workflows/`](.github/workflows/): ### Ephemeral Runner Orchestrator -GPU-required jobs (`runs-on: [self-hosted, airstack-ephemeral]`) execute on **OpenStack VMs spawned per-job and destroyed on completion**. The orchestrator service code lives in [`.github/orchestrator/`](.github/orchestrator/): +GPU-required jobs (`runs-on: [self-hosted, airstack-ephemeral]`) execute on **ephemeral pods scheduled by [NVIDIA OSMO](https://nvidia.github.io/OSMO/) — one per job, destroyed on completion**. The GitHub side is unchanged from the old OpenStack backend (same labels, JIT tokens, fork guard); only the spawn target moved from "create a Nova VM" to "submit an OSMO workflow". The orchestrator service code lives in [`.github/orchestrator/`](.github/orchestrator/): -- [`orchestrator.py`](.github/orchestrator/orchestrator.py) — Python service: spawn loop polls GitHub for queued jobs matching configured runner labels, mints single-use JIT runner tokens, creates an OpenStack server with cloud-init bootstrap; reap loop deletes the server when the job completes (or after `max_job_minutes`) -- [`cloud-init.yaml.j2`](.github/orchestrator/cloud-init.yaml.j2) — bootstraps Docker + nvidia-container-toolkit + GH Actions runner on the worker, registers with the JIT token, runs one job, then `shutdown -h` -- [`config.example.yaml`](.github/orchestrator/config.example.yaml) — flavor / network / keypair / floating-IP pool / runner labels / repo +- [`orchestrator.py`](.github/orchestrator/orchestrator.py) — Python service: spawn loop polls GitHub for queued jobs matching configured runner labels, mints single-use JIT runner tokens, and submits one OSMO workflow per job (`osmo workflow submit`); reap loop cancels the workflow when the job completes (or after `max_job_minutes`), plus an orphan sweep via `osmo workflow list` +- [`runner-workflow.yaml.j2`](.github/orchestrator/runner-workflow.yaml.j2) + [`runner.Dockerfile`](.github/orchestrator/runner.Dockerfile) + [`runner-entrypoint.sh`](.github/orchestrator/runner-entrypoint.sh) — the per-job worker: a **privileged**, GPU-enabled OSMO task (prebaked image) that starts an inner Docker daemon (the tests run `airstack up` = docker compose), registers with the JIT token, runs one job, then exits so OSMO reaps the pod +- [`config.example.yaml`](.github/orchestrator/config.example.yaml) — osmo_url / pool / platform / runner_image / resources / runner labels / repo - [`airstack-orchestrator.service`](.github/orchestrator/airstack-orchestrator.service) + [`setup.sh`](.github/orchestrator/setup.sh) — systemd unit and one-time installer -**Why ephemeral:** clean Docker cache per run, no leaked containers, GitHub PAT and OpenStack credentials only on the orchestrator host (workers receive a single-use JIT token bound to one runner registration). State map at `/var/lib/airstack-orchestrator/state.json`; logs via `journalctl -u airstack-orchestrator.service -f`. +**Why ephemeral:** clean Docker cache per run, no leaked containers; the GitHub PAT and the OSMO service-account token live only on the orchestrator host (workers receive a single-use JIT token bound to one runner registration). CI authenticates to OSMO as a shared, non-personal [service account](https://nvidia.github.io/OSMO/main/deployment_guide/appendix/authentication/service_accounts.html) scoped to a dedicated CI GPU pool, so runs don't consume individuals' quotas. The CI pool's platform must have **"Privileged Mode Allowed"** enabled (docker-in-docker). State map at `/var/lib/airstack-orchestrator/state.json`; logs via `journalctl -u airstack-orchestrator.service -f`. -**Setup, debugging a failed job, and SSH-into-worker procedures:** [`.github/orchestrator/README.md`](.github/orchestrator/README.md) (also exposed as [`tests/ci-cd-orchestrator.md`](tests/ci-cd-orchestrator.md) symlink for the docs site). +**Setup, debugging a failed job, and exec-into-worker procedures:** [`.github/orchestrator/README.md`](.github/orchestrator/README.md) (also exposed as [`tests/ci-cd-orchestrator.md`](tests/ci-cd-orchestrator.md) symlink for the docs site). ## Documentation Requirements diff --git a/tests/README.md b/tests/README.md index 6aec42e1b..aa87c0090 100644 --- a/tests/README.md +++ b/tests/README.md @@ -425,9 +425,9 @@ The workflow uses [`dawidd6/action-download-artifact@v6`](https://github.com/daw --- -## CI/CD Orchestrator (OpenStack-backed ephemeral runners) +## CI/CD Orchestrator (OSMO-backed ephemeral runners) -AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they execute on **truly ephemeral OpenStack instances** spawned per-job by an orchestrator. Each test job gets a fresh VM that is destroyed once the job completes — no Docker layer carryover, no leaked containers, no shared host state. +AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they execute on **truly ephemeral [NVIDIA OSMO](https://nvidia.github.io/OSMO/) pods** submitted per-job by an orchestrator. Each test job gets a fresh GPU pod that is destroyed once the job completes — no Docker layer carryover, no leaked containers, no shared host state. (This replaced an OpenStack-Nova backend; the GitHub side and the per-job-destroy model are unchanged.) ### Architecture @@ -436,19 +436,19 @@ AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they │ Orchestrator VM (airstack-ci-cd-orchestrator) │ │ • polls GitHub for queued workflow_jobs │ │ • mints single-use JIT runner tokens │ -│ • spawns / reaps ephemeral instances via OpenStack Nova │ -│ • holds the GitHub PAT and OpenStack application credential│ +│ • submits / reaps ephemeral OSMO workflows via osmo CLI │ +│ • holds the GitHub PAT and OSMO service-account token │ └────────────┬───────────────────────────────────┬─────────────┘ │ │ ▼ ▼ ┌──────────────────────────────┐ ┌────────────────────────────────┐ │ Ephemeral worker (per job) │ │ GitHub Actions queue │ -│ Image: Ubuntu-24.04-GPU- │ │ workflow_job status=queued │ -│ Headless │ │ labels: [self-hosted, │ -│ cloud-init bootstraps Docker │ │ airstack-ephemeral] │ -│ + nvidia-container-toolkit + │ └────────────────────────────────┘ -│ GH Actions runner; runs ONE │ -│ job, then is destroyed. │ +│ Prebaked airstack-ci-runner │ │ workflow_job status=queued │ +│ image: Docker + nvidia CTK + │ │ labels: [self-hosted, │ +│ GH runner. Privileged pod │ │ airstack-ephemeral] │ +│ starts dockerd, runs ONE │ └────────────────────────────────┘ +│ job (JIT), then the pod │ +│ is destroyed. │ └──────────────────────────────┘ ``` @@ -456,21 +456,22 @@ AirStack's tests require a GPU, Docker, and a clean filesystem per run, so they | Concern | Mitigation | |---------|------------| -| Cross-job state pollution (Docker cache, dangling networks, leftover artifacts) | Each job runs on a fresh VM. Spent VM is destroyed within ~30 s of job completion. | +| Cross-job state pollution (Docker cache, dangling networks, leftover artifacts) | Each job runs on a fresh OSMO pod, destroyed within ~30 s of job completion. | | Fork PRs executing arbitrary code | Workflow's `if: github.event.pull_request.head.repo.full_name == github.repository` — fork PRs skipped. | -| Runner running as root | The runner runs as the unprivileged `ubuntu` user inside an instance whose only purpose is one job. | -| Docker socket gives root-equivalent access | Bounded to a single one-shot VM. The orchestrator host doesn't expose Docker at all. | +| Runner runs privileged (root) for docker-in-docker | The pod is privileged (needed to run `airstack up`/compose), but it is single-use, scoped to the dedicated CI pool, and only same-repo code ever reaches it. | +| Docker socket gives root-equivalent access | Bounded to a single one-shot pod. The orchestrator host doesn't expose Docker at all. | | Long-lived PAT on the runner host | The PAT lives only on the orchestrator. Workers receive a single-use **JIT runner config** — a base64 token bound to one runner registration. | -| Persistent OpenStack creds tied to a user password | Orchestrator authenticates with an **application credential** (revocable, scoped) instead of `openrc.sh`. | +| Persistent creds tied to a personal account | Orchestrator authenticates with a shared, non-personal **OSMO service-account token** (revocable, scoped to the CI pool), not an individual's login. | ### Setup -The orchestrator service code, cloud-init template, systemd unit, and full setup runbook live in [`.github/orchestrator/`](../../../../.github/orchestrator/). See [`.github/orchestrator/README.md`](ci-cd-orchestrator.md) for: +The orchestrator service code, OSMO runner-workflow template, runner image, systemd unit, and full setup runbook live in [`.github/orchestrator/`](../../../../.github/orchestrator/). See [`.github/orchestrator/README.md`](ci-cd-orchestrator.md) for: -- creating the OpenStack application credential and `clouds.yaml` -- staging the GitHub PAT -- running `setup.sh` on the orchestrator VM -- filling in flavor / network / keypair / security-group in `/etc/airstack-orchestrator/config.yaml` +- obtaining the OSMO service-account token and a dedicated CI GPU pool (with privileged mode enabled) +- building and pushing the runner image (`runner.Dockerfile`) +- staging the GitHub PAT and the OSMO token +- running `setup.sh` on the orchestrator host (installs the `osmo` CLI) +- filling in osmo_url / pool / platform / runner_image / resources in `/etc/airstack-orchestrator/config.yaml` - enabling and verifying the `airstack-orchestrator.service` systemd unit ### Runner labels From 88d8c59a5fcc5ce59097efff85ec0668665cf2e5 Mon Sep 17 00:00:00 2001 From: pvkumara Date: Wed, 29 Jul 2026 14:26:31 -0400 Subject: [PATCH 12/27] ci(orchestrator): pin AirLab OSMO JSON keys and runner image path Resolve uuid/live name after submit (OSMO returns name-only + suffix), default config to the Keycloak-backed airstack pool and Harbor runner image, and add scripts to build/push airstack-ci-runner on OSMO DinD. Co-authored-by: Cursor --- .github/orchestrator/README.md | 21 +++++-- .github/orchestrator/build-and-push.sh | 28 +++++++++ .../orchestrator/build-runner-on-osmo.yaml | 63 +++++++++++++++++++ .github/orchestrator/config.example.yaml | 17 ++--- .github/orchestrator/orchestrator.py | 43 ++++++++++--- tests/README.md | 2 +- 6 files changed, 153 insertions(+), 21 deletions(-) create mode 100755 .github/orchestrator/build-and-push.sh create mode 100644 .github/orchestrator/build-runner-on-osmo.yaml diff --git a/.github/orchestrator/README.md b/.github/orchestrator/README.md index a1f0d7e94..1848ae2ad 100644 --- a/.github/orchestrator/README.md +++ b/.github/orchestrator/README.md @@ -67,13 +67,24 @@ The worker image bakes in Docker CE + compose, the NVIDIA container toolkit, and ```bash cd .github/orchestrator -docker build -f runner.Dockerfile \ - --build-arg RUNNER_VERSION=2.334.0 \ - -t /airstack-ci-runner:2.334.0 . -docker push /airstack-ci-runner:2.334.0 +./build-and-push.sh +# or manually: +# docker build -f runner.Dockerfile \ +# --build-arg RUNNER_VERSION=2.334.0 \ +# -t airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0 . +# docker push airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0 ``` -Set `runner_image: /airstack-ci-runner:2.334.0` in `config.yaml` (step 4). Keep `RUNNER_VERSION` in sync with an [actions/runner release](https://github.com/actions/runner/releases). +No local Docker? Submit the one-shot OSMO builder (needs your Harbor creds in OSMO): + +```bash +osmo workflow submit .github/orchestrator/build-runner-on-osmo.yaml \ + --pool airstack --priority HIGH +``` + +Set `runner_image: airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0` in `config.yaml` (step 4). Keep `RUNNER_VERSION` in sync with an [actions/runner release](https://github.com/actions/runner/releases). + +**Pool note (AirLab):** use the Keycloak-autosynced `airstack` pool (`privileged_allowed: true`). A hand-created `airstack-ci` pool is wiped by `synchronize_osmo_team_pools.py`. Ephemerality is per-job OSMO workflows, not a separate pool. ### 2. Stage credentials on the orchestrator host diff --git a/.github/orchestrator/build-and-push.sh b/.github/orchestrator/build-and-push.sh new file mode 100755 index 000000000..438112682 --- /dev/null +++ b/.github/orchestrator/build-and-push.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Build & push the AirStack CI ephemeral-runner image to AirLab Harbor. +# Run on a linux/amd64 machine (or buildx --platform linux/amd64) with: +# docker login airlab-docker.andrew.cmu.edu +# +# Usage: +# ./build-and-push.sh +# RUNNER_VERSION=2.334.0 ./build-and-push.sh + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY="${REGISTRY:-airlab-docker.andrew.cmu.edu/airstack}" +RUNNER_VERSION="${RUNNER_VERSION:-2.334.0}" +IMAGE="${REGISTRY}/airstack-ci-runner:${RUNNER_VERSION}" + +echo "==> Building ${IMAGE}" +docker build \ + -f "${ROOT}/runner.Dockerfile" \ + --build-arg "RUNNER_VERSION=${RUNNER_VERSION}" \ + -t "${IMAGE}" \ + "${ROOT}" + +echo "==> Pushing ${IMAGE}" +docker push "${IMAGE}" + +echo "==> Done. Set in /etc/airstack-orchestrator/config.yaml:" +echo " runner_image: \"${IMAGE}\"" diff --git a/.github/orchestrator/build-runner-on-osmo.yaml b/.github/orchestrator/build-runner-on-osmo.yaml new file mode 100644 index 000000000..98cb705d4 --- /dev/null +++ b/.github/orchestrator/build-runner-on-osmo.yaml @@ -0,0 +1,63 @@ +# One-shot OSMO job: build + push airstack-ci-runner to AirLab Harbor. +# Uses the existing privileged DinD workspace image (same as airstack-dev). +# +# Prereq: your OSMO profile has airlab-docker-login (+ auto REGISTRY cred). +# +# osmo workflow submit .github/orchestrator/build-runner-on-osmo.yaml \ +# --pool airstack --priority HIGH +# +# Watch: +# osmo workflow logs --task build +# Cancel when done if it hangs: +# osmo workflow cancel --force + +workflow: + name: build-airstack-ci-runner + resources: + build: + cpu: 8 + gpu: 0 + memory: 16Gi + storage: 100Gi + platform: default + tasks: + - name: build + image: airlab-docker.andrew.cmu.edu/airstack/airstack-osmo-workspace:latest + resource: build + privileged: true + credentials: + airlab-docker-login: + AIRLAB_REGISTRY_USER: username + AIRLAB_REGISTRY_PASS: password + environment: + AIRSTACK_REPO_URL: "https://github.com/castacks/AirStack.git" + AIRSTACK_BRANCH: "ci/osmo-orchestrator" + RUNNER_VERSION: "2.334.0" + REGISTRY: "airlab-docker.andrew.cmu.edu/airstack" + command: + - bash + - -lc + - | + set -euo pipefail + # Nested DinD: overlay-on-overlay breaks BuildKit. Use vfs + legacy builder. + mkdir -p /etc/docker + cat > /etc/docker/daemon.json <<'JSON' + {"storage-driver": "vfs"} + JSON + dockerd >/var/log/dockerd.log 2>&1 & + for _ in $(seq 1 90); do docker info >/dev/null 2>&1 && break; sleep 1; done + docker info >/dev/null 2>&1 || { cat /var/log/dockerd.log; exit 1; } + docker info | grep -i 'Storage Driver' || true + + echo "$AIRLAB_REGISTRY_PASS" | docker login airlab-docker.andrew.cmu.edu \ + -u "$AIRLAB_REGISTRY_USER" --password-stdin + + rm -rf /tmp/AirStack + git clone --depth 1 --branch "$AIRSTACK_BRANCH" "$AIRSTACK_REPO_URL" /tmp/AirStack + cd /tmp/AirStack/.github/orchestrator + IMAGE="${REGISTRY}/airstack-ci-runner:${RUNNER_VERSION}" + DOCKER_BUILDKIT=0 docker build -f runner.Dockerfile \ + --build-arg "RUNNER_VERSION=${RUNNER_VERSION}" \ + -t "$IMAGE" . + docker push "$IMAGE" + echo "PUSHED $IMAGE" diff --git a/.github/orchestrator/config.example.yaml b/.github/orchestrator/config.example.yaml index 27d8a3975..ab788edbb 100644 --- a/.github/orchestrator/config.example.yaml +++ b/.github/orchestrator/config.example.yaml @@ -8,7 +8,8 @@ osmo_bin: "osmo" # URL of your OSMO web service (the control plane the CLI logs into). -osmo_url: "https://osmo.example.com" +# AirLab: https://airlab-share-01.andrew.cmu.edu +osmo_url: "https://airlab-share-01.andrew.cmu.edu" # File containing the OSMO service-account access token. This is the shared, # non-personal "lab" identity — the analog of the old OpenStack application @@ -17,14 +18,16 @@ osmo_url: "https://osmo.example.com" # (see README). It never leaves this host. osmo_token_file: "/etc/airstack-orchestrator/osmo-token" -# Dedicated CI GPU pool. Give this pool its own allocation so CI runs don't -# compete with researchers' interactive quotas. The service-account's role must -# grant workflow:Create/Cancel/Query scoped to this pool (pool/). -pool: "airstack-ci" +# GPU pool for ephemeral runners. AirLab team pools are Keycloak-autosynced; +# use the stable `airstack` pool (privileged_allowed=true). A hand-made +# `airstack-ci` pool will be wiped by synchronize_osmo_team_pools.py unless +# it is added to Keycloak. The service-account needs workflow:Create on this +# pool (role osmo-airstack) plus osmo-user for cancel/query. +pool: "airstack" # Optional platform (hardware type) to target within the pool. Leave empty to # use the pool's default platform. List options with `osmo pool list` / the UI. -platform: "" +platform: "default" # Scheduling priority: HIGH | NORMAL | LOW. priority: "NORMAL" @@ -33,7 +36,7 @@ priority: "NORMAL" # Prebaked image: Docker CE + compose + nvidia-container-toolkit + GH Actions # runner. Build & push runner.Dockerfile to a registry the pool can pull from. -runner_image: "/airstack-ci-runner:2.334.0" +runner_image: "airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0" # Resource request for the runner container. Size for the full stack build + # sim (Isaac Sim / ms-airsim + robot + gcs) running under docker compose. diff --git a/.github/orchestrator/orchestrator.py b/.github/orchestrator/orchestrator.py index 5af8ea540..4af62555b 100644 --- a/.github/orchestrator/orchestrator.py +++ b/.github/orchestrator/orchestrator.py @@ -182,7 +182,12 @@ def get_job_status(repo: str, job_id: str, pat: str) -> dict | None: # deployed version with `osmo workflow submit --dry-run` / `--format-type json` # once and simplify if desired. -_WF_ID_KEYS = ("workflow_id", "workflowId", "id", "uuid", "name", "workflow") +# Live AirLab OSMO 6.2.x returns workflow_uuid on list/query; submit may use +# workflow_id / id / name. Prefer uuid-like keys before "name" so we don't +# accidentally treat the human workflow name as the id when both are present. +_WF_ID_KEYS = ( + "workflow_uuid", "workflow_id", "workflowId", "id", "uuid", "name", "workflow", +) _STATUS_KEYS = ("status", "state", "workflow_status", "phase") _KNOWN_STATUSES = { "RUNNING", "PENDING", "WAITING", "COMPLETED", "FAILED", @@ -382,7 +387,14 @@ def _osmo(self, args: list[str], timeout: int, r = self._run_osmo(args, timeout=timeout) return r - def submit_workflow(self, workflow_file: str) -> str: + def submit_workflow(self, workflow_file: str) -> tuple[str, str]: + """Submit a workflow. Returns (workflow_id, live_name). + + AirLab OSMO 6.2 submit JSON is typically only {name, overview, logs} + (no uuid), and the service may append a numeric suffix to the name + (e.g. ``...-1``). We immediately query to resolve uuid + live name so + state/reap stay consistent with ``workflow list`` (``workflow_uuid``). + """ args = ["workflow", "submit", workflow_file, "--pool", self.pool, "--priority", self.priority, "--format-type", "json"] r = self._osmo(args, timeout=self.submit_timeout) @@ -391,13 +403,28 @@ def submit_workflow(self, workflow_file: str) -> str: f"osmo workflow submit failed (rc={r.returncode}): " f"{(r.stderr or r.stdout).strip()}" ) - wid = _extract_workflow_id(r.stdout) or _extract_workflow_id(r.stderr) - if not wid: + submitted_name = _extract_workflow_id(r.stdout) or _extract_workflow_id(r.stderr) + if not submitted_name: raise RuntimeError( - "could not parse workflow id from submit output: " + "could not parse workflow id/name from submit output: " f"{(r.stdout or '').strip()[:500]}" ) - return wid + live_name, uuid = submitted_name, None + q = self._osmo( + ["workflow", "query", submitted_name, "--format-type", "json"], + timeout=60, + ) + if q.returncode == 0: + data = _loads_or_none(q.stdout) or _loads_or_none(q.stderr) + if isinstance(data, dict): + if isinstance(data.get("name"), str) and data["name"]: + live_name = data["name"] + for k in ("uuid", "workflow_uuid", "workflow_id", "id"): + v = data.get(k) + if isinstance(v, str) and v: + uuid = v + break + return (uuid or live_name), live_name def query_status(self, workflow_id: str) -> str | None: r = self._osmo( @@ -489,7 +516,7 @@ def spawn_once(self) -> None: ) workflow_yaml = self.render_workflow(workflow_name, jit) tmp_path = self._write_temp_workflow(workflow_name, workflow_yaml) - workflow_id = self.submit_workflow(tmp_path) + workflow_id, live_name = self.submit_workflow(tmp_path) except Exception as e: # noqa: BLE001 log.exception("submit failed for job %s: %s", job_id, e) continue @@ -503,7 +530,7 @@ def spawn_once(self) -> None: state["jobs"][job_id] = { "run_id": job["run_id"], "workflow_id": workflow_id, - "workflow_name": workflow_name, + "workflow_name": live_name, "runner_name": workflow_name, "submitted_at": now_utc_iso(), "name": job["name"], diff --git a/tests/README.md b/tests/README.md index aa87c0090..a7b24b1c8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -409,7 +409,7 @@ Regressions are flagged with :red_circle:, improvements with :green_circle:. #### Jobs -**`run-tests`** runs on a freshly-spawned ephemeral OpenStack instance (`[self-hosted, airstack-ephemeral]`). The instance is provisioned per-job by the orchestrator described below and destroyed once the job completes. It installs dependencies, runs pytest, and uploads `tests/results/` as an artifact named `test-results--` with 90-day retention. +**`run-tests`** runs on a freshly-spawned ephemeral OSMO pod (`[self-hosted, airstack-ephemeral]`). The pod is submitted per-job by the orchestrator described below and destroyed once the job completes. It installs dependencies, runs pytest, and uploads `tests/results/` as an artifact named `test-results--` with 90-day retention. **`report`** runs on `ubuntu-latest` after `run-tests` (even if it failed). It: From 165d01138a6413a6770edf9a3bbfacc0bcca1552 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Fri, 7 Aug 2026 13:36:05 -0400 Subject: [PATCH 13/27] docs(ci): document the OSMO-backed CI/CD pipeline Fills in the empty ci_cd.md stub with an end-to-end guide to how CI runs the full AirStack stack on ephemeral OSMO GPU pods: architecture and job lifecycle diagrams, runner pod anatomy, the three trigger paths, what each pytest mark catches, the metrics regression gate, the security model, and layer-by-layer troubleshooting. Adds the page to the mkdocs nav (it was previously unreachable) and cross-links it from tests/README.md and the testing index. Co-authored-by: Cursor --- .../development/intermediate/testing/ci_cd.md | 493 +++++++++++++++++- .../development/intermediate/testing/index.md | 2 +- mkdocs.yml | 1 + tests/README.md | 5 + 4 files changed, 499 insertions(+), 2 deletions(-) diff --git a/docs/development/intermediate/testing/ci_cd.md b/docs/development/intermediate/testing/ci_cd.md index a5afaf265..a60619fb0 100644 --- a/docs/development/intermediate/testing/ci_cd.md +++ b/docs/development/intermediate/testing/ci_cd.md @@ -1 +1,492 @@ -# CI/CD Pipeline \ No newline at end of file +# CI/CD Pipeline on OSMO + +AirStack's continuous integration runs the **full drone stack** — simulator, +robot autonomy, and GCS — on a GPU for every change. Because that needs a +GPU, a Docker daemon, and a clean filesystem, jobs cannot run on GitHub's +hosted runners and should not run on a shared always-on machine. Instead, a +small orchestrator service watches the GitHub Actions queue and submits one +**ephemeral [NVIDIA OSMO](https://nvidia.github.io/OSMO/) pod per job**. The pod +registers as a single-use GitHub Actions runner, executes exactly one job, and +is destroyed. + +This page documents the whole system: the architecture, the job lifecycle, +what each test suite actually catches, how to trigger and read a run, and how +to fit CI into your day-to-day development loop. + +!!! note "Related pages" + - [`tests/README.md`](../../../../tests/README.md) — the test suite reference: marks, fixtures, metrics, CLI flags. + - [CI/CD Orchestrator](../../../../tests/ci-cd-orchestrator.md) — the lab-admin runbook: pool prerequisites, credential staging, `setup.sh`, rotation, break-glass debugging. + - [AirStack on OSMO](../../../tutorials/airstack_on_osmo.md) — the *interactive* OSMO dev pod (Remote-SSH + Isaac Sim streaming). Different workflow, same compute pool. + +--- + +## The short version + +| Question | Answer | +|---|---| +| Where do CI jobs run? | A fresh GPU pod on the OSMO `airstack` pool, one per job, destroyed after. | +| What triggers a run? | A PR being **opened**, a `/pytest` comment from a maintainer, or manual `workflow_dispatch`. | +| What gets tested? | Docker image builds, `colcon` builds, unit tests, stack liveliness, sensor rates, takeoff/hover/land, fixed-trajectory tracking. | +| How do I see results? | A metrics report comment on the PR, plus the `test-results-*` artifact (`summary.txt`, `results.xml`, `metrics.json`). | +| What fails the build? | Any failed test, **or** a metric regressing more than 20 % against the base branch's last run. | +| Who holds the secrets? | Only the orchestrator host. Workers get a single-use JIT token valid for one registration. | + +--- + +## Architecture + +Three planes, each owning one job. GitHub owns the queue and the logs. The +orchestrator owns the credentials and the job ↔ pod bookkeeping. OSMO owns the +GPU compute and the pod lifecycle. + +```mermaid +flowchart LR + subgraph gh [GitHub] + pr["Pull request / comment / dispatch"] + queue["Actions queue
workflow_job: queued
labels: self-hosted, airstack-ephemeral"] + api["REST API"] + pr --> queue + queue --- api + end + + subgraph orch ["Orchestrator host (no GPU, always on)"] + svc["airstack-orchestrator.service
orchestrator.py"] + spawn["spawn loop — every 15s"] + reap["reap loop — every 30s"] + creds["/etc/airstack-orchestrator
github-pat + osmo-token + config.yaml"] + state["/var/lib/airstack-orchestrator/state.json
job_id → workflow_id"] + svc --> spawn + svc --> reap + svc --- creds + spawn --- state + reap --- state + end + + subgraph osmo ["OSMO airstack pool (GPU, privileged)"] + wf["Workflow gha-runner-JOBID-TS"] + pod["Ephemeral runner pod
airstack-ci-runner image"] + wf --> pod + end + + api -- "poll queued jobs" --> spawn + spawn -- "mint JIT runner config" --> api + spawn -- "osmo workflow submit" --> wf + pod -- "register + long-poll for work" --> api + reap -- "osmo workflow cancel" --> wf +``` + +Key properties that fall out of this shape: + +- **Truly ephemeral.** Every job starts from the prebaked image with an empty Docker cache. No leftover containers, no dangling networks, no "works because the last run left something behind". +- **PAT isolation.** The GitHub PAT never leaves the orchestrator. The pod receives a [JIT runner config](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository) — a base64 blob bound to exactly one runner registration, short-lived. +- **Non-personal OSMO identity.** The orchestrator authenticates with a service-account token scoped to the CI pool, so runs never consume an individual's GPU quota and nothing breaks when someone graduates. +- **Crash-safe.** Every workflow is named `gha-runner--`. The reap loop cancels any active workflow with that prefix that is missing from `state.json`, so a crashed or restarted orchestrator cannot leak pods. + +--- + +## Job lifecycle + +From "you comment `/pytest`" to "the pod is gone", in order: + +```mermaid +sequenceDiagram + autonumber + participant Dev as Developer + participant GH as GitHub Actions + participant Orch as Orchestrator + participant OSMO as OSMO scheduler + participant Pod as Runner pod + + Dev->>GH: open PR / comment /pytest / dispatch + GH->>GH: queue job with labels self-hosted + airstack-ephemeral + Orch->>GH: poll queued jobs (15s) + Orch->>GH: POST generate-jitconfig + GH-->>Orch: encoded_jit_config (single use) + Orch->>Orch: render runner-workflow.yaml.j2 + Orch->>OSMO: osmo workflow submit --pool airstack + OSMO-->>Orch: workflow name, then uuid via query + Orch->>Orch: record job_id to workflow_id in state.json + OSMO->>Pod: schedule privileged GPU pod + Pod->>Pod: start inner dockerd, nvidia-smi check + Pod->>GH: run.sh --jitconfig, register ephemeral runner + GH->>Pod: dispatch the one job + Pod->>Pod: checkout, pull images, pytest + Pod-->>GH: logs, conclusion, results artifact + Pod->>Pod: run.sh exits after one job, task completes + OSMO->>OSMO: tear the pod down + Orch->>GH: poll job status (30s) + GH-->>Orch: completed + Orch->>OSMO: cancel if still live, then drop from state.json +``` + +Two safety nets run on top of the happy path: + +- **Straggler reap.** Any tracked job older than `max_job_minutes` (default 48 h) is force-cancelled regardless of what GitHub reports. +- **Orphan sweep.** Active `gha-runner-*` workflows that are not in `state.json` and are more than two minutes old get cancelled. The two-minute grace window prevents the sweep from racing a submit that has not been recorded yet. + +--- + +## Anatomy of a runner pod + +The worker is a prebaked image — everything the job needs is already in the +layer cache when the pod starts, so a slow `apt-get` can never outlive the JIT +token's validity window. + +```mermaid +flowchart TB + subgraph pod ["OSMO task — privileged, 1 GPU, 8 CPU, 32Gi RAM, 300Gi disk"] + entry["run-ephemeral-runner.sh"] + dockerd["inner dockerd
+ nvidia-container-toolkit"] + runner["actions-runner run.sh --jitconfig"] + subgraph compose ["docker compose stack started by airstack up"] + sim["isaac-sim or ms-airsim"] + robot["robot-desktop x NUM_ROBOTS"] + gcs["gcs"] + end + entry --> dockerd + entry --> runner + runner -- "pytest tests/ runs airstack up" --> dockerd + dockerd --> sim + dockerd --> robot + dockerd --> gcs + end + gpu["Node GPU"] --> dockerd +``` + +| Piece | File | What it contributes | +|---|---|---| +| Image | [`runner.Dockerfile`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner.Dockerfile) | Ubuntu 24.04 + Docker CE + compose/buildx + NVIDIA container toolkit + pinned `actions/runner` (2.334.0) | +| Entrypoint | [`runner-entrypoint.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-entrypoint.sh) | Starts `dockerd`, waits up to 60 s for it, runs `nvidia-smi` as a non-fatal GPU sanity check, then `exec`s `run.sh --jitconfig` | +| Pod shape | [`runner-workflow.yaml.j2`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-workflow.yaml.j2) | Resource request, `privileged: true`, the JIT config and `RUNNER_ALLOW_RUNASROOT` env | +| Sizing | `config.yaml` | `cpu: 8`, `gpu: 1`, `memory: 32Gi`, `storage: 300Gi` — sized for sim + robot + GCS images plus Isaac assets | + +!!! warning "Privileged is mandatory" + The tests run `airstack up`, which is `docker compose`, which needs a Docker + daemon *inside* the pod. That requires the pool's platform to have + **Privileged Mode Allowed** enabled. Without it, submissions are rejected and + `osmo workflow logs` shows `dockerd did not become ready`. + +Build and publish the image with +[`build-and-push.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/build-and-push.sh), +or — if you have no local Docker — submit +[`build-runner-on-osmo.yaml`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/build-runner-on-osmo.yaml), +a one-shot OSMO job that builds the runner image inside an OSMO pod and pushes +it to Harbor. + +--- + +## Triggering a run + +### The three entry points + +| Trigger | When it fires | What it runs | +|---|---|---| +| `pull_request` (`types: [opened]`) | Only when the PR is first opened, and only for same-repo branches | pytest's `conftest` defaults — the full mark set | +| `/pytest` PR comment | Any time, from a user with `OWNER`/`MEMBER`/`COLLABORATOR` association | Whatever args you put on the first line of the comment | +| `workflow_dispatch` | Manual, from the Actions tab | The form inputs: `marks`, `sim`, `num_robots`, `stress_iterations`, `stable_duration`, `baseline_run_id` | + +Pushes to an open PR deliberately do **not** re-trigger. GPU pods are a shared +resource, so re-runs are opt-in via `/pytest`. + +### Comment syntax + +The first line is parsed with `shlex`; everything after it is free-form notes. + +```text +/pytest -m liveliness --sim msairsim --num-robots 1 --stress-iterations 1 + +Checking whether the DDS bridge fix holds under 3 robots — see thread above. +``` + +The workflow replies on the thread with the exact `pytest` command it resolved +and a link to the run, and opens a **Check Run** pinned to the PR head SHA so +comment-triggered runs still show up in the PR's Checks tab. + +!!! tip "`build_packages` is prepended for you" + Whenever you pass `-m`, the workflow rewrites the expression to + `build_packages or `. Launch tests are useless against a stale + `install/` tree, and this removes the most common way to waste a 40-minute + GPU run. It is skipped when you already named `build_packages`, and when you + pass no marks at all (pytest then runs everything anyway). + +### What the job does, step by step + +```mermaid +flowchart TD + a["Resolve PR head — issue_comment only"] --> b["Parse pytest args
prepend build_packages, extract --sim"] + b --> c["Ack comment + open in-progress Check Run"] + c --> d["Checkout PR head with submodules"] + d --> e["Write omni_pass.env — guest Nucleus creds"] + e --> f["Create venv, install tests/requirements.txt"] + f --> g{"Registry secrets present?"} + g -- yes --> h["docker login, set AIRSTACK_REGISTRY_CACHE=1"] + g -- no --> i["Skip — build from scratch"] + h --> j{"marks contain build_docker?"} + i --> j + j -- yes --> l["Skip image prep — those tests build themselves"] + j -- no --> k["airstack image-pull for the active profiles
fall back to image-build for anything missing"] + k --> m["pytest tests/ with resolved args"] + l --> m + m --> n["Upload tests/results/ artifact, 90-day retention"] + n --> o["Finalize Check Run with the job conclusion"] + o --> p["report job on ubuntu-latest"] +``` + +The image-prep step is what makes runs on a cold pod tolerable: it pulls the +published images for exactly the compose profiles the selected `--sim` implies, +then falls back to a local build only for images the registry did not have (a +new branch that has not been released yet, for example). + +--- + +## What the pipeline tests, and what that catches + +Tests are selected with pytest marks. Collection order is fixed in +`tests/conftest.py` so cheap and prerequisite suites always run first — a +`colcon` break fails in minutes instead of after a sim bring-up. + +```mermaid +flowchart LR + u["unit
seconds, no Docker"] --> bd["build_docker
image builds"] + bd --> bp["build_packages
colcon build in containers"] + bp --> lv["liveliness
stack comes up"] + lv --> sn["sensors
streams flow at rate"] + sn --> th["takeoff_hover_land
flight chain"] + th --> au["autonomy
trajectory tracking"] +``` + +| Mark | Module | What it verifies | Bugs it is good at catching | +|---|---|---|---| +| `unit` | `tests/robot/`, `tests/sim/` proxies | Hermetic Python/numpy logic co-located with each ROS 2 package | Off-by-one and boundary errors in filters, converters, validators; regressions in pure algorithm code | +| `build_docker` | `system/test_build_docker.py` | Every image builds; records image sizes | Broken Dockerfiles, deleted apt packages, upstream base-image drift, accidental image bloat | +| `build_packages` | `system/test_build_packages.py` | `colcon build` inside robot, GCS, and ms-airsim workspaces | Missing `package.xml` dependencies, uninstalled launch/config files, C++ breakage on a clean tree | +| `liveliness` | `system/test_liveliness.py` | Containers reach Running, `/clock` publishes, tmux panes alive, sentinel ROS 2 nodes present, compute snapshot, stability poll | Launch files that crash on start, nodes that die after 30 s, `ROBOT_NAME`/domain-ID misconfiguration, runaway CPU or memory | +| `sensors` | `system/test_sensors.py` | Stereo and depth publish rates on both sim and robot side, filtered LiDAR liveness plus geometry sanity, sim real-time factor, time-series stability | Broken sim-to-ROS bridges, sensor Hz that silently halves, RTF collapse from a heavy new node, LiDAR filter range regressions | +| `takeoff_hover_land` | `system/test_takeoff_hover_land.py` | Four-phase chain per (sim, robots, iteration, velocity): PX4 ready → takeoff to 10 m → hover → land | Controller tuning regressions, altitude overshoot, hover drift, state-estimation bias against ground truth, PX4/MAVROS handshake breakage | +| `autonomy` | `system/test_fixed_trajectory.py` | Same chain with a Circle / Figure8 / Racetrack / Line pattern in the middle; records cross-track error and path RMSE | Path-tracker regressions, trajectory-library math errors, velocity/acceleration limit violations that show up as corner-cutting | + +### The flight chain + +Both flight suites run as an ordered chain per parametrization, so the drone +always ends on the ground before the next configuration starts: + +```mermaid +flowchart LR + r["test_px4_ready
MAVROS + EKF"] --> t["test_takeoff
within 10% of 10 m"] + t --> x["test_hover or test_fixed_trajectory"] + x --> l["test_landing
final altitude < 0.5 m"] + r -. "failure" .-> s["remaining phases skipped"] + t -. "failure" .-> s + x -. "failure still lands" .-> l +``` + +A failure in the middle phase (`test_hover` or `test_fixed_trajectory`) does +**not** skip landing — a bad tracker must not leave a drone stuck in the air +blocking the rest of the sweep. A failure in `test_px4_ready` or `test_takeoff` +does skip the remaining phases for that configuration. + +### Bring-up scope, and why mark selection costs money + +`airstack_env` is **class-scoped** and parametrized over +`(sim, num_robots, iteration)`. Each test class does its own `airstack up` and +`airstack down`. Selecting two suites with `or` therefore performs **two full +stack cycles per tuple**: + +```text +-m liveliness → 1 bring-up per (sim, robots, iter) +-m "liveliness or sensors" → 2 bring-ups per (sim, robots, iter) +--sim msairsim,isaacsim → doubles all of the above +--num-robots 1,3 → doubles it again +``` + +Run one mark at a time unless you genuinely need both. + +--- + +## Reading the results + +### The PR comment + +After `run-tests` finishes — pass or fail — a `report` job on `ubuntu-latest` +downloads the current artifact plus a **baseline** artifact and runs +[`parse_metrics.py`](https://github.com/castacks/AirStack/blob/main/tests/parse_metrics.py) +in diff mode. + +| Run type | Baseline used | +|---|---| +| PR opened or `/pytest` | Latest `system-tests.yml` artifact on the PR's base branch | +| `workflow_dispatch` with `baseline_run_id` | That specific run | +| `workflow_dispatch` without it | Latest artifact on `main` | + +The comment has three sections per test module: a flat **Metrics** table, a +**Sim publishing rates** pivot (topic Hz aggregates from the `sensors` mark), +and a **Compute usage** pivot (CPU / memory / GPU per container). Regressions +are marked with a red circle, improvements with a green one, and the job +**fails** if any metric moves more than the 20 % threshold in the wrong +direction. That is the mechanism that catches slow degradation — the kind of +change where nothing throws but the tracker is quietly 30 % worse. + +### The artifact + +`test-results--`, retained 90 days: + +```text +tests/results/2026-08-06_14-30-00/ +├── summary.txt # human-readable per-chain summary — open this first +├── results.xml # JUnit XML: durations, pass/fail per test +└── metrics.json # every recorded metric, including time series +``` + +There are no per-test log files. Live output streams to the Actions log via +pytest's `log_cli`, and failed assertions embed the tail of the relevant +`docker` or `ros2` subprocess output directly in the failure message. + +Regenerate a report locally from a downloaded artifact: + +```bash +python tests/parse_metrics.py \ + --current path/to/current-run/ \ + --baseline path/to/baseline-run/ \ + --threshold 20 +``` + +--- + +## Using CI well while developing + +The pipeline is expensive at the far end and nearly free at the near end. Push +each class of failure as far left as it will go. + +```mermaid +flowchart TD + q{"What did you change?"} + q -- "Pure Python / numpy logic" --> u["airstack test -m unit
seconds, no GPU"] + q -- "Dockerfile / dependency" --> b["airstack test -m build_docker or build_packages
minutes, no GPU"] + q -- "Launch file / new node" --> l["airstack test -m liveliness --sim msairsim --num-robots 1"] + q -- "Sensor or bridge" --> s["airstack test -m sensors --sim isaacsim --num-robots 1"] + q -- "Controller / planner" --> a["airstack test -m autonomy --sim msairsim --trajectory-types Circle"] + u --> pr["Push branch, open PR"] + b --> pr + l --> pr + s --> pr + a --> pr + pr --> ci["Full suite runs on the ephemeral GPU pod"] + ci --> rep["Read the metrics comment"] + rep --> iter["/pytest with a narrowed mark to confirm a fix"] +``` + +Practical rules that follow from how the system is built: + +- **Reproduce CI locally with the same command.** `airstack test` and CI both call `pytest tests/` with the same flags. If a run fails in CI, copy the resolved command from the acknowledgment comment and run it on any GPU box — including an [interactive OSMO dev pod](../../../tutorials/airstack_on_osmo.md) if you do not have a local GPU. +- **Narrow before you re-run.** A `/pytest` with no args re-runs everything. `/pytest -m autonomy --sim msairsim --trajectory-types Circle` re-runs the one chain you are fixing, in a fraction of the time. +- **Never trust a green launch test against a stale build.** This is why `build_packages` is auto-prepended; keep it that way when writing your own `/pytest` line. +- **Read `summary.txt` before the raw log.** It groups each flight chain with per-phase wall times and status, so the failing phase is obvious without scrolling a 40-minute log. +- **Treat the metrics diff as a review artifact.** A PR that turns a metric red needs an explanation in the thread, even when every test passed. +- **Bump `VERSION` in `.env` when image content changes.** [`check-version-increment.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/check-version-increment.yml) gates the PR on a strictly-greater semver, and merging that bump is what triggers the release build below. + +--- + +## The release path + +`system-tests.yml` is not the only workflow on the ephemeral runners. +[`docker-build.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/docker-build.yml) +also requests `runs-on: [self-hosted, airstack-ephemeral]` and therefore gets +the same per-job pod treatment. + +```mermaid +flowchart LR + pr["PR merged to main or develop"] --> chk{".env VERSION changed?"} + chk -- no --> stop["No build"] + chk -- yes --> pod["Ephemeral OSMO pod"] + pod --> build["docker compose build"] + build --> push["docker compose push"] + push --> sign["cosign sign — keyless, GitHub OIDC"] + sign --> verify["cosign verify against the workflow identity"] +``` + +Signing is keyless via GitHub's OIDC token, and the same job immediately +verifies each pushed digest against the expected certificate identity, so a +published image that was not built by this workflow fails the check. + +| Workflow | Runner | Purpose | +|---|---|---| +| `system-tests.yml` | Ephemeral OSMO GPU pod | Full test suite + metrics report | +| `docker-build.yml` | Ephemeral OSMO GPU pod | Build, push, and sign all compose images | +| `check-version-increment.yml` | `ubuntu-latest` | Semver gate on `.env` `VERSION=` | +| `deploy_docs_from_*.yaml` | `ubuntu-latest` | Versioned MkDocs publish via `mike` | + +--- + +## Security model + +| Concern | How the design handles it | +|---|---| +| Cross-job state pollution | Fresh pod per job with an empty Docker cache; destroyed within ~30 s of completion | +| Fork PRs executing arbitrary code on a GPU node | `head.repo.full_name == github.repository` guard on the `pull_request` path, and an explicit fork check plus `author_association` gate on the `/pytest` path | +| Long-lived GitHub PAT on a worker | The PAT lives only on the orchestrator; workers get a single-use JIT config bound to one registration | +| Credentials tied to a person | OSMO auth uses a non-personal service-account token scoped to the CI pool | +| Privileged container is root-equivalent | Accepted deliberately — docker-in-docker is required — but bounded to a one-shot pod, on a dedicated pool, running only same-repo code | +| Orchestrator compromise blast radius | Systemd hardening: `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome=read-only`, `PrivateTmp`, with a single `ReadWritePaths` for state | +| Leaked pods after a crash | Name-prefix orphan sweep plus a `max_job_minutes` straggler ceiling | + +--- + +## Troubleshooting + +A failed run can break at the orchestrator, at the OSMO pod, at the runner, or +in the tests themselves, and each layer has a different inspection path. Work +down the list. + +| Symptom | Layer | First thing to check | +|---|---|---| +| Job sits `queued` forever, no pod appears | Orchestrator | `journalctl -u airstack-orchestrator.service --since '30 min ago'` — look for `submitted workflow for job ` | +| `find_queued_jobs failed: 401` | Orchestrator | GitHub PAT expired or lost a scope; rotate it | +| `osmo login failed` / auth error | Orchestrator | OSMO service-account token expired (default 31 days); mint a new one and restart the service | +| `osmo workflow submit failed ... privileged` | OSMO pool | The pool's platform lacks **Privileged Mode Allowed** | +| Job queued but never claimed | Labels | `runs-on` labels must be a superset of `runner_labels` in `config.yaml` | +| `dockerd did not become ready` | Pod | Not actually privileged; check the platform, then `osmo workflow logs "$WF" --task runner` | +| `nvidia-smi unavailable` | Pod | GPU not requested or the toolkit is not configured on the node | +| `Cannot connect to the Docker daemon` mid-test | Pod | Inner dockerd crashed — `osmo workflow exec "$WF" runner`, then read `/var/log/dockerd.log` | +| `No space left on device` | Pod | Bump `storage` in `config.yaml`; Isaac assets plus all images are large | +| Runner registered, then pytest failed | Tests | A real test failure — the GitHub Actions log and `summary.txt` are canonical | +| Metrics report job failed with no test failures | Report | A metric regressed past the 20 % threshold; read the diff table | + +To map a GitHub job to its pod: + +```bash +JOB_ID=73286176852 # from the GitHub Actions URL +WF=$(sudo jq -r ".jobs[\"$JOB_ID\"].workflow_id" /var/lib/airstack-orchestrator/state.json) + +osmo workflow query "$WF" --verbose +osmo workflow events "$WF" --task runner # scheduling, image pull, eviction +osmo workflow logs "$WF" --task runner # dockerd, run.sh, and the job itself +osmo workflow exec "$WF" runner # break-glass shell, while RUNNING +``` + +Full runbook, including credential rotation and worker-side diagnostics: +[CI/CD Orchestrator](../../../../tests/ci-cd-orchestrator.md). + +--- + +## File map + +| Path | Role | +|---|---| +| [`.github/workflows/system-tests.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/system-tests.yml) | The test workflow: triggers, arg parsing, image prep, pytest, artifact, metrics report | +| [`.github/orchestrator/orchestrator.py`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/orchestrator.py) | The spawn and reap loops, GitHub polling, JIT minting, OSMO CLI plumbing | +| [`.github/orchestrator/runner-workflow.yaml.j2`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-workflow.yaml.j2) | Per-job OSMO workflow template | +| [`.github/orchestrator/runner.Dockerfile`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner.Dockerfile) | Prebaked worker image | +| [`.github/orchestrator/runner-entrypoint.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/runner-entrypoint.sh) | dockerd bring-up, GPU check, single-job runner | +| [`.github/orchestrator/config.example.yaml`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/config.example.yaml) | Every tunable: pool, platform, resources, limits, poll intervals | +| [`.github/orchestrator/setup.sh`](https://github.com/castacks/AirStack/blob/main/.github/orchestrator/setup.sh) | One-time orchestrator host install | +| [`tests/conftest.py`](https://github.com/castacks/AirStack/blob/main/tests/conftest.py) | `airstack_env` fixture, collection order, `MetricsRecorder` | +| [`tests/parse_metrics.py`](https://github.com/castacks/AirStack/blob/main/tests/parse_metrics.py) | Report generation and the regression gate | +| [`tests/run_summary.py`](https://github.com/castacks/AirStack/blob/main/tests/run_summary.py) | `summary.txt` generation | + +## See also + +- [System Tests](../../../../tests/README.md) — marks, fixtures, metrics, and every CLI flag. +- [Unit Testing](unit_testing.md) — the co-location and proxy pattern for package-level tests. +- [End-to-End Testing](end_to_end_testing.md) — the fixed-trajectory benchmark in depth. +- [CI/CD Orchestrator](../../../../tests/ci-cd-orchestrator.md) — admin setup, rotation, and break-glass procedures. +- [AirStack on OSMO](../../../tutorials/airstack_on_osmo.md) — interactive GPU dev pods on the same pool. diff --git a/docs/development/intermediate/testing/index.md b/docs/development/intermediate/testing/index.md index e6ce09c0b..087617018 100644 --- a/docs/development/intermediate/testing/index.md +++ b/docs/development/intermediate/testing/index.md @@ -86,4 +86,4 @@ airstack test -m "build_packages or autonomy" \ - [Unit Testing](unit_testing.md) — `@pytest.mark.unit`, proxy pattern, CI workflow - [Testing frameworks](testing_frameworks.md) — `colcon test`, rostest patterns - [Integration testing](integration_testing.md) -- [CI/CD](ci_cd.md) — pipeline overview +- [CI/CD Pipeline on OSMO](ci_cd.md) — how CI runs the full stack on ephemeral GPU pods: architecture, triggers, what each mark catches, and the metrics regression gate diff --git a/mkdocs.yml b/mkdocs.yml index e75d85bc7..c4845657e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,6 +70,7 @@ nav: - Unit Testing: docs/development/intermediate/testing/unit_testing.md - System Tests: tests/README.md - End-to-End Testing: docs/development/intermediate/testing/end_to_end_testing.md + - CI/CD Pipeline: docs/development/intermediate/testing/ci_cd.md - CI/CD Orchestrator: tests/ci-cd-orchestrator.md - Frame Conventions: docs/development/intermediate/frame_conventions.md - Docker Build Profiles: docs/development/intermediate/docker-build-profiles.md diff --git a/tests/README.md b/tests/README.md index a7b24b1c8..e9b9ea620 100644 --- a/tests/README.md +++ b/tests/README.md @@ -389,6 +389,11 @@ Regressions are flagged with :red_circle:, improvements with :green_circle:. ## CI/CD Integration +!!! note "Full pipeline guide" + For the end-to-end picture — architecture diagrams, job lifecycle, trigger + reference, what each mark catches, and how to fold CI into your development + loop — see **[CI/CD Pipeline on OSMO](../docs/development/intermediate/testing/ci_cd.md)**. + ### Workflow: `system-tests.yml` [`.github/workflows/system-tests.yml`](../../../../.github/workflows/system-tests.yml) runs on: From f56810dca581bf16b6fb272979139d9342cb86c8 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Fri, 7 Aug 2026 14:23:44 -0400 Subject: [PATCH 14/27] fix(ci): repair Docker builds on OSMO ephemeral runners Every build_docker and build_packages test failed on the OSMO backend because the inner dockerd kept its data-root on the pod's overlayfs rootfs. Linux rejects a directory on overlayfs as an overlay upperdir, so image pulls still succeeded -- containerd unpacks layers with plain writes -- while every build step needing a real mount died with "mount source: overlay ... err: invalid argument", surfacing as unrelated-looking apt-get and WORKDIR failures. runner-entrypoint.sh now picks a storage backend by attempting a real overlay mount rather than trusting the filesystem type, preferring a loopback ext4 data-root (real overlay2, sparse, dies with the pod) and falling back to a pod-mounted filesystem, fuse-overlayfs, then vfs. vfs is a last resort only: it copies the whole filesystem per layer and would exhaust the storage request on the sim images. Also bumps the GitHub Actions runner to 2.336.0, since 2.334.0 stops being able to run jobs on 2026-08-10. Co-authored-by: Cursor --- .github/orchestrator/README.md | 47 ++++++ .github/orchestrator/build-and-push.sh | 4 +- .../orchestrator/build-runner-on-osmo.yaml | 2 +- .github/orchestrator/config.example.yaml | 4 +- .github/orchestrator/runner-entrypoint.sh | 141 +++++++++++++++++- .github/orchestrator/runner.Dockerfile | 15 +- AGENTS.md | 8 + 7 files changed, 210 insertions(+), 11 deletions(-) diff --git a/.github/orchestrator/README.md b/.github/orchestrator/README.md index 1848ae2ad..f2df1ff22 100644 --- a/.github/orchestrator/README.md +++ b/.github/orchestrator/README.md @@ -257,3 +257,50 @@ nvidia-smi | `Cannot connect to the Docker daemon` during tests | inner dockerd crashed | Read `/var/log/dockerd.log` via `osmo workflow exec` | | Runner registered, then `pytest` failed | A normal test failure | Read the GitHub Actions log — the canonical view | | `No space left on device` | `storage` too small for images + sim assets | Bump `storage` in `config.yaml` | +| `failed to solve: ... mount source: "overlay" ... err: invalid argument` | Docker data-root landed on the pod's overlay rootfs | See "Nested DinD and overlayfs" below | + +### Nested DinD and overlayfs + +The single most likely way to break every Docker build at once. The pod's root +filesystem is overlayfs, and **Linux refuses to use a directory on overlayfs as +an overlay `upperdir`** (it returns `EINVAL`). A dockerd whose data-root sits on +the pod rootfs looks healthy — `docker info` works, image pulls succeed, because +containerd unpacks layers with plain writes — and then every build step that +needs a real mount fails: + +```text +failed to solve: process "/bin/bash -c apt-get ... " did not complete successfully: +mount source: "overlay", +target: "/var/lib/docker/buildkit/containerd-overlayfs/cachemounts/buildkit1459786452", +fstype: overlay, ... err: invalid argument +``` + +That signature took out all four `build_docker` tests and all four +`build_packages` tests in one run, with each failure looking like an unrelated +`apt-get`/`WORKDIR` problem. + +[`runner-entrypoint.sh`](runner-entrypoint.sh) handles this before starting +dockerd. It picks a storage backend by **performing a real overlay mount** to +test each option rather than trusting the filesystem type, and falls back in +this order: + +| Order | Backend | Notes | +|---|---|---| +| 1 | Loopback ext4 image mounted at `/var/lib/docker` | Preferred. Real `overlay2`, self-contained, dies with the pod. Sparse, so it only consumes what Docker writes. Sized to free space on `/` minus 20 GiB, or `DOCKER_LOOP_SIZE_MB`. | +| 2 | A real filesystem already mounted in the pod | Kubernetes `emptyDir`/`hostPath`/PVC volumes live on the node disk, not the overlay rootfs. `/osmo/data/output` and `/osmo/data/socket` are skipped — the OSMO ctrl sidecar owns them. | +| 3 | `fuse-overlayfs` driver | Stacks where the kernel driver won't. Needs `/dev/fuse`. | +| 4 | `vfs` driver | Always works, copies the whole filesystem per layer. Too slow and too large for the sim images — **reaching this is a red flag**, not a working state. | + +The chosen backend is logged at startup, so confirm it in the job log before +debugging anything else: + +```bash +osmo workflow logs "$WF" --task runner | grep -E 'runner-entrypoint|storage driver' +``` + +Backends 3 and 4 also set `features.containerd-snapshotter: false`, because +`storage-driver` is only honoured by the classic image store. + +The one-shot [`build-runner-on-osmo.yaml`](build-runner-on-osmo.yaml) builder +sidesteps the same problem differently — `vfs` plus `DOCKER_BUILDKIT=0` — which +is fine there because it builds one small image. diff --git a/.github/orchestrator/build-and-push.sh b/.github/orchestrator/build-and-push.sh index 438112682..aacefb002 100755 --- a/.github/orchestrator/build-and-push.sh +++ b/.github/orchestrator/build-and-push.sh @@ -5,13 +5,13 @@ # # Usage: # ./build-and-push.sh -# RUNNER_VERSION=2.334.0 ./build-and-push.sh +# RUNNER_VERSION=2.336.0 ./build-and-push.sh set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REGISTRY="${REGISTRY:-airlab-docker.andrew.cmu.edu/airstack}" -RUNNER_VERSION="${RUNNER_VERSION:-2.334.0}" +RUNNER_VERSION="${RUNNER_VERSION:-2.336.0}" IMAGE="${REGISTRY}/airstack-ci-runner:${RUNNER_VERSION}" echo "==> Building ${IMAGE}" diff --git a/.github/orchestrator/build-runner-on-osmo.yaml b/.github/orchestrator/build-runner-on-osmo.yaml index 98cb705d4..a44f21ab5 100644 --- a/.github/orchestrator/build-runner-on-osmo.yaml +++ b/.github/orchestrator/build-runner-on-osmo.yaml @@ -32,7 +32,7 @@ workflow: environment: AIRSTACK_REPO_URL: "https://github.com/castacks/AirStack.git" AIRSTACK_BRANCH: "ci/osmo-orchestrator" - RUNNER_VERSION: "2.334.0" + RUNNER_VERSION: "2.336.0" REGISTRY: "airlab-docker.andrew.cmu.edu/airstack" command: - bash diff --git a/.github/orchestrator/config.example.yaml b/.github/orchestrator/config.example.yaml index ab788edbb..ba1c64384 100644 --- a/.github/orchestrator/config.example.yaml +++ b/.github/orchestrator/config.example.yaml @@ -36,7 +36,7 @@ priority: "NORMAL" # Prebaked image: Docker CE + compose + nvidia-container-toolkit + GH Actions # runner. Build & push runner.Dockerfile to a registry the pool can pull from. -runner_image: "airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.334.0" +runner_image: "airlab-docker.andrew.cmu.edu/airstack/airstack-ci-runner:2.336.0" # Resource request for the runner container. Size for the full stack build + # sim (Isaac Sim / ms-airsim + robot + gcs) running under docker compose. @@ -60,7 +60,7 @@ host_network: false # reference/traceability; it is a build arg of runner.Dockerfile, not consumed # by the orchestrator at runtime. Must match a tag at # https://github.com/actions/runner/releases -runner_version: "2.334.0" +runner_version: "2.336.0" # --- GitHub --- diff --git a/.github/orchestrator/runner-entrypoint.sh b/.github/orchestrator/runner-entrypoint.sh index cc5696ae8..3026dbdf4 100644 --- a/.github/orchestrator/runner-entrypoint.sh +++ b/.github/orchestrator/runner-entrypoint.sh @@ -10,10 +10,146 @@ # Requires a privileged pod (dockerd) scheduled on a GPU platform; the NVIDIA # container toolkit (baked into the image) lets the inner dockerd pass the node # GPU through to the compose containers. -set -euxo pipefail +set -euo pipefail : "${ENCODED_JIT_CONFIG:?ENCODED_JIT_CONFIG must be set by the workflow}" +log() { echo "[runner-entrypoint] $*"; } + +# --------------------------------------------------------------------------- +# Docker storage backend +# +# The pod's root filesystem is overlayfs, and Linux refuses to use a directory +# on overlayfs as an overlay `upperdir` (EINVAL). A dockerd whose data-root sits +# on the pod rootfs still *pulls* images fine — containerd unpacks layers with +# plain writes — but every build step that needs a real mount dies with: +# +# failed to solve: ... mount source: "overlay", +# target: "/var/lib/docker/buildkit/containerd-overlayfs/cachemounts/...", +# err: invalid argument +# +# which is what took out all of build_docker / build_packages. So put the +# data-root somewhere overlay actually works, and verify it by performing a real +# overlay mount rather than trusting the filesystem type. +# --------------------------------------------------------------------------- + +DOCKER_DATA_ROOT=/var/lib/docker +LOOP_IMG=/docker-data.img +# Headroom left for everything that is not the Docker data-root (the runner's +# _work checkout, logs, the loop image's own metadata). +LOOP_HEADROOM_MB=20480 +MIN_BACKING_MB=51200 # Below ~50 GiB the sim images can't fit regardless. + +# True if an overlay mount whose upperdir lives under $1 can actually be made. +overlay_upperdir_works() { + local base=$1 probe rc=1 + probe=$(mktemp -d "$base/.overlay-probe.XXXXXX" 2>/dev/null) || return 1 + mkdir -p "$probe"/{lower,upper,work,merged} + if mount -t overlay overlay \ + -o "lowerdir=$probe/lower,upperdir=$probe/upper,workdir=$probe/work" \ + "$probe/merged" 2>/dev/null; then + umount "$probe/merged" && rc=0 + fi + rm -rf "$probe" + return $rc +} + +free_mb() { df -Pm "$1" | awk 'NR==2 {print $4}'; } + +# Preferred: a loopback ext4 image mounted at the data-root. Self-contained +# (no dependency on how the pool exposes storage), gives real overlay2, and dies +# with the pod. The image file is sparse, so it only consumes what Docker writes. +setup_loopback() { + local size_mb=${DOCKER_LOOP_SIZE_MB:-} + if [[ -z "$size_mb" ]]; then + size_mb=$(( $(free_mb /) - LOOP_HEADROOM_MB )) + fi + if (( size_mb < MIN_BACKING_MB )); then + log "loopback: only ${size_mb}MB usable, need ${MIN_BACKING_MB}MB — skipping" + return 1 + fi + + # /dev/loop-control only exists once the loop module is loaded on the node. + [[ -e /dev/loop-control ]] || modprobe loop 2>/dev/null || true + if [[ ! -e /dev/loop-control ]]; then + log "loopback: no /dev/loop-control — skipping" + return 1 + fi + + log "loopback: creating ${size_mb}MB ext4 image at $LOOP_IMG" + truncate -s "${size_mb}M" "$LOOP_IMG" || return 1 + # No journal and lazy inode-table init: this filesystem never outlives the + # pod, so durability buys nothing and mkfs stays fast. + mkfs.ext4 -q -F -m 0 -O ^has_journal -E lazy_itable_init=1 "$LOOP_IMG" || return 1 + + mkdir -p "$DOCKER_DATA_ROOT" + mount -o loop "$LOOP_IMG" "$DOCKER_DATA_ROOT" || return 1 + + if ! overlay_upperdir_works "$DOCKER_DATA_ROOT"; then + log "loopback: mounted but overlay still rejected — unwinding" + umount "$DOCKER_DATA_ROOT" || true + rm -f "$LOOP_IMG" + return 1 + fi + return 0 +} + +# Fallback: a real filesystem already mounted into the pod. Kubernetes emptyDir, +# hostPath and PVC volumes are backed by the node disk rather than the overlay +# rootfs, so overlay works there. +setup_real_fs() { + local best="" best_free=0 mnt fstype opts avail + while read -r _ mnt fstype opts _; do + case "$fstype" in ext2|ext3|ext4|xfs|btrfs) ;; *) continue ;; esac + [[ -d "$mnt" && -w "$mnt" ]] || continue + [[ ",$opts," == *",ro,"* ]] && continue + # OSMO's ctrl sidecar owns these: /osmo/data/output is uploaded as job + # artifacts and the socket dir is its IPC channel. + case "$mnt" in /osmo/data/output*|/osmo/data/socket*) continue ;; esac + avail=$(free_mb "$mnt") + if (( avail > best_free )); then best_free=$avail; best=$mnt; fi + done < /proc/mounts + + if [[ -z "$best" ]] || (( best_free < MIN_BACKING_MB )); then + log "real-fs: no mounted filesystem with >=${MIN_BACKING_MB}MB free — skipping" + return 1 + fi + + DOCKER_DATA_ROOT="$best/airstack-docker-data" + mkdir -p "$DOCKER_DATA_ROOT" + if ! overlay_upperdir_works "$DOCKER_DATA_ROOT"; then + log "real-fs: overlay rejected under $best — skipping" + DOCKER_DATA_ROOT=/var/lib/docker + return 1 + fi + log "real-fs: using $DOCKER_DATA_ROOT (${best_free}MB free)" + return 0 +} + +mkdir -p /etc/docker +if setup_loopback; then + log "storage: overlay2 on a loopback ext4 image" + printf '{"data-root": "%s"}\n' "$DOCKER_DATA_ROOT" > /etc/docker/daemon.json +elif setup_real_fs; then + log "storage: overlay2 on $DOCKER_DATA_ROOT" + printf '{"data-root": "%s"}\n' "$DOCKER_DATA_ROOT" > /etc/docker/daemon.json +elif [[ -e /dev/fuse ]] && command -v fuse-overlayfs >/dev/null 2>&1; then + # fuse-overlayfs stacks on overlayfs where the kernel driver won't. Slower + # than overlay2 but nowhere near as bad as vfs. `storage-driver` only applies + # to the classic image store, so the containerd snapshotter has to go. + log "storage: fuse-overlayfs (no overlay-capable filesystem found)" + cat > /etc/docker/daemon.json <<'JSON' +{"storage-driver": "fuse-overlayfs", "features": {"containerd-snapshotter": false}} +JSON +else + # Always works, but copies the whole filesystem per layer. The sim images are + # large enough that this will likely exhaust the pod's storage request. + log "WARN: storage: falling back to vfs — builds will be slow and may run out of disk" + cat > /etc/docker/daemon.json <<'JSON' +{"storage-driver": "vfs", "features": {"containerd-snapshotter": false}} +JSON +fi + # Start dockerd in the background (needs privileged). dockerd >/var/log/dockerd.log 2>&1 & @@ -30,6 +166,9 @@ if ! docker info >/dev/null 2>&1; then exit 1 fi +log "storage driver: $(docker info --format '{{.Driver}}' 2>/dev/null || echo unknown)" \ + "data-root: $(docker info --format '{{.DockerRootDir}}' 2>/dev/null || echo unknown)" + # Non-fatal GPU sanity check — surfaces GPU/privileged/toolkit misconfig early. nvidia-smi || echo "WARN: nvidia-smi unavailable (check GPU + privileged + toolkit)" diff --git a/.github/orchestrator/runner.Dockerfile b/.github/orchestrator/runner.Dockerfile index 05af72566..18bcf8000 100644 --- a/.github/orchestrator/runner.Dockerfile +++ b/.github/orchestrator/runner.Dockerfile @@ -6,11 +6,11 @@ # a slow apt/bootstrap. Build it and push to a registry your OSMO pool can pull: # # docker build -f runner.Dockerfile \ -# --build-arg RUNNER_VERSION=2.334.0 \ -# -t /airstack-ci-runner:2.334.0 . -# docker push /airstack-ci-runner:2.334.0 +# --build-arg RUNNER_VERSION=2.336.0 \ +# -t /airstack-ci-runner:2.336.0 . +# docker push /airstack-ci-runner:2.336.0 # -# Then set `runner_image: /airstack-ci-runner:2.334.0` in config.yaml. +# Then set `runner_image: /airstack-ci-runner:2.336.0` in config.yaml. # Keep RUNNER_VERSION in sync with the actions/runner release you want. # # GPU-in-Docker-in-Docker: the OSMO task must run privileged (see @@ -24,8 +24,13 @@ ENV DEBIAN_FRONTEND=noninteractive # Docker CE (+ compose/buildx plugins), NVIDIA container toolkit, and the tools # the AirStack test harness / GH runner need (git, jq, python venv, ...). +# e2fsprogs + fuse-overlayfs back the storage-backend selection in +# runner-entrypoint.sh: the pod rootfs is overlayfs, which the kernel rejects as +# an overlay upperdir, so dockerd's data-root has to live on a loopback ext4 +# image (mkfs.ext4) or, failing that, use the fuse-overlayfs driver. RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates curl gnupg jq git sudo iproute2 \ + e2fsprogs fuse-overlayfs kmod mount \ python3 python3-venv python3-pip \ && install -m 0755 -d /etc/apt/keyrings \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ @@ -47,7 +52,7 @@ https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_C && rm -rf /var/lib/apt/lists/* # GitHub Actions runner (self-contained; version pinned at build time). -ARG RUNNER_VERSION=2.334.0 +ARG RUNNER_VERSION=2.336.0 RUN mkdir -p /home/runner/actions-runner \ && cd /home/runner/actions-runner \ && curl -fsSL -o runner.tar.gz \ diff --git a/AGENTS.md b/AGENTS.md index 4207c2b87..a261cc63f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -277,6 +277,14 @@ GPU-required jobs (`runs-on: [self-hosted, airstack-ephemeral]`) execute on **ep **Why ephemeral:** clean Docker cache per run, no leaked containers; the GitHub PAT and the OSMO service-account token live only on the orchestrator host (workers receive a single-use JIT token bound to one runner registration). CI authenticates to OSMO as a shared, non-personal [service account](https://nvidia.github.io/OSMO/main/deployment_guide/appendix/authentication/service_accounts.html) scoped to a dedicated CI GPU pool, so runs don't consume individuals' quotas. The CI pool's platform must have **"Privileged Mode Allowed"** enabled (docker-in-docker). State map at `/var/lib/airstack-orchestrator/state.json`; logs via `journalctl -u airstack-orchestrator.service -f`. +**Nested DinD needs a non-overlayfs Docker data-root.** The OSMO pod's root filesystem is overlayfs, and Linux rejects a directory on overlayfs as an overlay `upperdir` (`EINVAL`). A dockerd storing data on the pod rootfs pulls images fine but fails every build step that needs a real mount, with errors that masquerade as `apt-get`/`WORKDIR` failures: + +``` +failed to solve: ... mount source: "overlay", target: ".../buildkit/containerd-overlayfs/cachemounts/...", err: invalid argument +``` + +[`runner-entrypoint.sh`](.github/orchestrator/runner-entrypoint.sh) picks a backend by attempting a real overlay mount, preferring a loopback ext4 image at `/var/lib/docker` (real `overlay2`), then a real filesystem already mounted in the pod, then `fuse-overlayfs`, then `vfs`. Landing on `vfs` means builds will be slow and probably run out of disk — check the `[runner-entrypoint] storage:` line in the job log first when Docker builds misbehave. Details: [orchestrator README → Nested DinD and overlayfs](.github/orchestrator/README.md). + **Setup, debugging a failed job, and exec-into-worker procedures:** [`.github/orchestrator/README.md`](.github/orchestrator/README.md) (also exposed as [`tests/ci-cd-orchestrator.md`](tests/ci-cd-orchestrator.md) symlink for the docs site). ## Documentation Requirements From 4583165626d9e29902bc2203e908e45d1b9ef684 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Fri, 7 Aug 2026 17:10:23 -0400 Subject: [PATCH 15/27] fix(ci): seed PR Docker builds from a floating cache tag Versioned cache_from entries always miss on PRs because VERSION is forced up; add a stable cache_* tag published only by docker-build.yml so system tests can reuse layers without writing the shared cache. Co-authored-by: Cursor --- .github/workflows/docker-build.yml | 34 ++++++++++ .github/workflows/system-tests.yml | 4 ++ AGENTS.md | 2 + airstack.sh | 64 +++++++++++++++++-- .../development/intermediate/testing/ci_cd.md | 39 ++++++++++- gcs/docker/gcs-base-docker-compose.yaml | 2 + robot/docker/docker-compose.yaml | 16 +++++ .../isaac-sim/docker/docker-compose.yaml | 5 ++ .../ms-airsim/docker/docker-compose.yaml | 2 + 9 files changed, 161 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index add5f5953..b415635fd 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -131,6 +131,40 @@ jobs: docker compose push + # `docker compose push` only publishes each service's `image:` (the + # versioned tag). PR builds can never hit that tag as cache, because + # check-version-increment forces VERSION up on every PR — so they also + # cache_from a floating CACHE_TAG that only this workflow republishes. + # `docker compose build` already applied these via `build.tags`; they + # point at the digest just pushed, so Cosign's digest-based signature + # covers them too. + - name: Publish floating cache tags + run: | + set -a + source .env + set +a + + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" + else + export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" + fi + + TAGS=$(docker compose config --format json \ + | jq -r --arg pfx ":${CACHE_TAG:-cache}_" \ + '.services[].build.tags // [] | .[] | select(contains($pfx))' \ + | sort -u) + + if [ -z "$TAGS" ]; then + echo "No cache tags resolved from compose config; nothing to publish." + exit 1 + fi + + for TAG in $TAGS; do + echo "Pushing cache tag $TAG" + docker push "$TAG" + done + - name: Sign pushed images with Cosign (keyless) env: COSIGN_YES: "true" diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 3ddebda58..f60240a44 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -268,6 +268,10 @@ jobs: # inline cache (build_docker tests get layer-reuse speedup) and pre-pulls # before `airstack up` (other tests skip the implicit rebuild). When # secrets are absent both steps are skipped and behavior is unchanged. + # + # Read-only on purpose: AIRSTACK_REGISTRY_CACHE_PUSH stays unset here so a + # PR can consume the floating cache tag but never republish it. Only + # docker-build.yml (main/develop) writes it. - name: Log in to internal Docker registry id: docker_login if: ${{ vars.DOCKER_REGISTRY_URL != '' && env.DOCKER_REGISTRY_PASSWORD != '' }} diff --git a/AGENTS.md b/AGENTS.md index a261cc63f..e19877727 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,6 +285,8 @@ failed to solve: ... mount source: "overlay", target: ".../buildkit/containerd-o [`runner-entrypoint.sh`](.github/orchestrator/runner-entrypoint.sh) picks a backend by attempting a real overlay mount, preferring a loopback ext4 image at `/var/lib/docker` (real `overlay2`), then a real filesystem already mounted in the pod, then `fuse-overlayfs`, then `vfs`. Landing on `vfs` means builds will be slow and probably run out of disk — check the `[runner-entrypoint] storage:` line in the job log first when Docker builds misbehave. Details: [orchestrator README → Nested DinD and overlayfs](.github/orchestrator/README.md). +**Docker layer cache is a floating tag, not the versioned one.** Every compose service lists two `cache_from` entries: the versioned image (`airstack:v${VERSION}_`) and a floating one (`airstack:${CACHE_TAG:-cache}_`). Only the floating tag can ever hit on a PR — `check-version-increment` forces `VERSION` up on every PR, so the versioned tag it builds under has by definition never been pushed. Reading and writing are separate switches: `AIRSTACK_REGISTRY_CACHE=1` (set by `system-tests.yml`) pulls and builds with `BUILDKIT_INLINE_CACHE=1`, while `AIRSTACK_REGISTRY_CACHE_PUSH=1` (set only by `docker-build.yml` on main/develop) also publishes both tags. PR runs stay read-only so an unmerged branch can't poison the shared cache or publish an unreleased version. If you add a service with a `build:` section, give it both entries or its builds will always be cold. + **Setup, debugging a failed job, and exec-into-worker procedures:** [`.github/orchestrator/README.md`](.github/orchestrator/README.md) (also exposed as [`tests/ci-cd-orchestrator.md`](tests/ci-cd-orchestrator.md) symlink for the docs site). ## Documentation Requirements diff --git a/airstack.sh b/airstack.sh index 1c57c47cb..a0a1d431c 100755 --- a/airstack.sh +++ b/airstack.sh @@ -824,6 +824,46 @@ function ensure_robot_l4t_stack_base() { run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${_ga[@]}" build "${build_opts[@]}" robot-l4t-stack-base } +# `docker compose push` only publishes each service's `image:`. The floating +# cache tags are declared in `build.tags`, so they need an explicit push. Read +# them back out of the resolved config instead of reconstructing the names here, +# so this stays correct as services are added. +function push_cache_tags() { + local -n _ga="$1" + local -n _sc="$2" + + if ! command -v jq >/dev/null 2>&1; then + log_warn "jq not found; skipping cache-tag push (floating cache will go stale)" + return 0 + fi + + # Service names only — drop any flags that were passed through to the subcommand. + local services=() + for arg in "${_sc[@]}"; do + [[ "$arg" == -* ]] || services+=("$arg") + done + + local tags + tags=$(run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${_ga[@]}" config --format json 2>/dev/null \ + | jq -r --arg svcs "${services[*]}" --arg pfx ":${CACHE_TAG:-cache}_" ' + .services | to_entries[] + | select($svcs == "" or (($svcs | split(" ")) | index(.key))) + | (.value.build.tags // [])[] + | select(contains($pfx)) + ' 2>/dev/null | sort -u) + + if [[ -z "$tags" ]]; then + log_warn "No cache tags resolved from compose config; nothing to publish" + return 0 + fi + + while IFS= read -r tag; do + [[ -n "$tag" ]] || continue + log_info "Pushing cache tag $tag" + docker push "$tag" || log_warn "Failed to push cache tag $tag" + done <<< "$tags" +} + function cmd_up { check_docker @@ -877,9 +917,16 @@ function cmd_image_build { # Registry-cache mode (CI / opt-in): pre-pull existing images to seed the # local cache, build with BUILDKIT_INLINE_CACHE=1 so the resulting image - # carries layer-cache metadata, and push so the next run benefits. The - # cache_from declarations in each component compose file make BuildKit - # actually reuse the pulled layers. No-op when the env var is unset. + # carries layer-cache metadata. The cache_from declarations in each + # component compose file make BuildKit actually reuse the pulled layers. + # No-op when the env var is unset. + # + # Reading and publishing the cache are separate switches. A PR bumps VERSION + # (check-version-increment enforces it), so the versioned cache_from entry is + # guaranteed to miss and the floating CACHE_TAG entry is what actually hits. + # PR runs must not write that floating tag: an unmerged branch would poison + # the shared cache and publish an unreleased VERSION. Only trusted branches + # set AIRSTACK_REGISTRY_CACHE_PUSH=1. if [[ "${AIRSTACK_REGISTRY_CACHE:-}" == "1" ]]; then log_info "AIRSTACK_REGISTRY_CACHE=1 → pulling for cache seed..." run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" pull --ignore-pull-failures "${subcmd_args[@]}" || \ @@ -888,9 +935,14 @@ function cmd_image_build { log_info "Building services with BUILDKIT_INLINE_CACHE=1..." run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" build --build-arg BUILDKIT_INLINE_CACHE=1 "${subcmd_args[@]}" - log_info "Pushing built images for next-run cache..." - run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" push --ignore-push-failures "${subcmd_args[@]}" || \ - log_warn "Post-build push encountered failures; future runs may not benefit from cache" + if [[ "${AIRSTACK_REGISTRY_CACHE_PUSH:-}" == "1" ]]; then + log_info "Pushing built images for next-run cache..." + run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" push --ignore-push-failures "${subcmd_args[@]}" || \ + log_warn "Post-build push encountered failures; future runs may not benefit from cache" + push_cache_tags global_args subcmd_args + else + log_info "AIRSTACK_REGISTRY_CACHE_PUSH is not 1 → cache is read-only for this run" + fi else log_info "Building services..." run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" build "${subcmd_args[@]}" diff --git a/docs/development/intermediate/testing/ci_cd.md b/docs/development/intermediate/testing/ci_cd.md index a60619fb0..175b4c8e1 100644 --- a/docs/development/intermediate/testing/ci_cd.md +++ b/docs/development/intermediate/testing/ci_cd.md @@ -219,7 +219,7 @@ flowchart TD d --> e["Write omni_pass.env — guest Nucleus creds"] e --> f["Create venv, install tests/requirements.txt"] f --> g{"Registry secrets present?"} - g -- yes --> h["docker login, set AIRSTACK_REGISTRY_CACHE=1"] + g -- yes --> h["docker login, set AIRSTACK_REGISTRY_CACHE=1
read-only: PRs never republish the cache"] g -- no --> i["Skip — build from scratch"] h --> j{"marks contain build_docker?"} i --> j @@ -237,6 +237,43 @@ published images for exactly the compose profiles the selected `--sim` implies, then falls back to a local build only for images the registry did not have (a new branch that has not been released yet, for example). +### Layer cache: the floating `cache_*` tag + +Every pod starts with an empty Docker cache, so `build_docker` is only fast if +BuildKit can import layers from the registry. Each compose service therefore +declares two `cache_from` entries: + +| Entry | Example | Who writes it | +|---|---|---| +| Versioned | `airstack:v0.19.0-alpha.7_isaac-sim` | `docker-build.yml`, per release | +| Floating | `airstack:cache_isaac-sim` | `docker-build.yml`, republished every build | + +The versioned entry alone cannot work on a pull request. `check-version-increment` +requires every PR to raise `VERSION`, so the tag a PR builds under is by +definition one that has never been pushed — the pull misses and the build runs +cold from the first `RUN` layer: + +``` +Image ...airstack:v0.19.0-alpha.7_isaac-sim Pulling +Image ...airstack:v0.19.0-alpha.7_isaac-sim failed to resolve reference +``` + +The floating tag is the one that actually hits. It tracks the newest build from +`main`/`develop` rather than any particular version, so a PR imports the layers +its base branch already produced and rebuilds only what it changed. + +Reading and writing the cache are separate switches, and PR runs get read only: + +- `AIRSTACK_REGISTRY_CACHE=1` — pull to seed, build with `BUILDKIT_INLINE_CACHE=1`. + Set by `system-tests.yml` whenever registry secrets are available. +- `AIRSTACK_REGISTRY_CACHE_PUSH=1` — additionally publish the versioned and + floating tags. Set only by `docker-build.yml`. + +Keeping the write switch off for pull requests means an unmerged branch can +neither poison the shared cache for everyone else nor publish an unreleased +`VERSION` tag. Override the tag name with `CACHE_TAG` (default `cache`) to keep +an experimental cache line separate. + --- ## What the pipeline tests, and what that catches diff --git a/gcs/docker/gcs-base-docker-compose.yaml b/gcs/docker/gcs-base-docker-compose.yaml index c8ac5200e..8894bcb9f 100644 --- a/gcs/docker/gcs-base-docker-compose.yaml +++ b/gcs/docker/gcs-base-docker-compose.yaml @@ -7,8 +7,10 @@ services: dockerfile: docker/Dockerfile.gcs tags: - &gcs_image ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:v${VERSION}_gcs + - &gcs_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_gcs cache_from: - *gcs_image + - *gcs_cache command: > bash -c " ssh service restart; diff --git a/robot/docker/docker-compose.yaml b/robot/docker/docker-compose.yaml index 31b3e368a..cc2bbb6c2 100644 --- a/robot/docker/docker-compose.yaml +++ b/robot/docker/docker-compose.yaml @@ -17,8 +17,13 @@ services: ROS_DISTRO: jazzy tags: - *desktop_image + # Floating tag republished by docker-build.yml on main/develop. The + # versioned tag above never exists yet on a PR (check-version-increment + # forces VERSION up), so it can only ever be a cache miss. + - &desktop_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-x86-64_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *desktop_image + - *desktop_cache # we use tmux sd-keys so that the session stays alive environment: - ROBOT_NAME_SOURCE=container_name # see .bashrc @@ -126,8 +131,10 @@ services: ROS_DISTRO: jazzy tags: - *voxl_image + - &voxl_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-voxl_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *voxl_image + - *voxl_cache environment: - ROBOT_NAME_SOURCE=hostname # see .bashrc - AUTOLAUNCH=${AUTOLAUNCH:-true} @@ -158,8 +165,10 @@ services: DUSTYNV_IMAGE: dustynv/ros:jazzy-ros-base-r36.4.0-cu128-24.04 tags: - *l4t_stack_base_image + - &l4t_stack_base_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-l4t-stack-base_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *l4t_stack_base_image + - *l4t_stack_base_cache # =================================================================================================================== # for running on an NVIDIA jetson (linux for tegra) device @@ -184,9 +193,12 @@ services: ROS_DISTRO: jazzy tags: - *l4t_image + - &l4t_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_robot-l4t_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *l4t_image + - *l4t_cache - *l4t_stack_base_image + - *l4t_stack_base_cache # we use tmux send-keys so that the session stays alive ipc: host command: > @@ -246,8 +258,12 @@ services: L4T_MINOR: 4 L4T_PATCH: 0 IMAGE_NAME: dustynv/ros:jazzy-desktop-r36.4.0-cu128-24.04 + tags: + - *zed_l4t_image + - &zed_l4t_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_zed-l4t-36-4-0_${DOCKER_IMAGE_BUILD_MODE} cache_from: - *zed_l4t_image + - *zed_l4t_cache command: > bash -c "ssh service restart; tmux new -d -s zed_driver && diff --git a/simulation/isaac-sim/docker/docker-compose.yaml b/simulation/isaac-sim/docker/docker-compose.yaml index dfd699aa4..50a9df16d 100644 --- a/simulation/isaac-sim/docker/docker-compose.yaml +++ b/simulation/isaac-sim/docker/docker-compose.yaml @@ -8,8 +8,13 @@ services: dockerfile: docker/Dockerfile.isaac-ros tags: - *image_tag + # Floating tag republished by docker-build.yml on main/develop. The + # versioned tag above never exists yet on a PR (check-version-increment + # forces VERSION up), so it can only ever be a cache miss. + - &cache_tag ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_isaac-sim cache_from: - *image_tag + - *cache_tag container_name: isaac-sim entrypoint: "" command: > diff --git a/simulation/ms-airsim/docker/docker-compose.yaml b/simulation/ms-airsim/docker/docker-compose.yaml index 3a4035374..9b49c9098 100644 --- a/simulation/ms-airsim/docker/docker-compose.yaml +++ b/simulation/ms-airsim/docker/docker-compose.yaml @@ -8,8 +8,10 @@ services: dockerfile: Dockerfile tags: - *ms_airsim_image + - &ms_airsim_cache ${PROJECT_DOCKER_REGISTRY}/${PROJECT_NAME}:${CACHE_TAG:-cache}_ms-airsim cache_from: - *ms_airsim_image + - *ms_airsim_cache container_name: ms-airsim entrypoint: "" command: /root/entrypoint.sh From 5550b76ec7468bf688b84d19668f23051e6ab82c Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Fri, 7 Aug 2026 17:24:16 -0400 Subject: [PATCH 16/27] ci(docker-build): retag unchanged images on VERSION bump Skip full compose rebuilds when a service's content fingerprint matches the previous versioned image label; registry-retag instead and only rebuild services whose Docker inputs changed. Co-authored-by: Cursor --- .../skills/bump-version-and-release/SKILL.md | 12 +- .github/workflows/docker-build.yml | 234 +++++--- .../workflows/scripts/docker_image_plan.py | 528 ++++++++++++++++++ .gitignore | 5 + AGENTS.md | 2 + .../development/intermediate/testing/ci_cd.md | 44 +- 6 files changed, 741 insertions(+), 84 deletions(-) create mode 100755 .github/workflows/scripts/docker_image_plan.py diff --git a/.agents/skills/bump-version-and-release/SKILL.md b/.agents/skills/bump-version-and-release/SKILL.md index 3c056b774..18792a965 100644 --- a/.agents/skills/bump-version-and-release/SKILL.md +++ b/.agents/skills/bump-version-and-release/SKILL.md @@ -64,11 +64,13 @@ Three workflows in `.github/workflows/` interact with `VERSION`: - **Trigger:** push to `main` or `develop` whose changed paths include `.env`, **and** the `VERSION=` line in `.env` differs from the previous commit. Also runs on manual `workflow_dispatch`. - **Behavior on tag change:** 1. Runs on a self-hosted ephemeral GPU runner (`[self-hosted, airstack-ephemeral]`). - 2. `docker compose build` for profiles `desktop,isaac-sim,ms-airsim`. - 3. `docker compose push` to `${PROJECT_DOCKER_REGISTRY}` (set in `.env` — currently `airlab-docker.andrew.cmu.edu/airstack`). - 4. Keyless `cosign sign` of every pushed image digest via GitHub OIDC. - 5. `cosign verify` against the workflow's certificate identity. + 2. Plans per service via `.github/workflows/scripts/docker_image_plan.py` (content fingerprint vs previous versioned image label). + 3. **Unchanged image inputs** → registry retag of the previous `v${PREV}_…` digest to `v${VERSION}_…` and `cache_*` (no rebuild). + 4. **Changed inputs** (or missing/unlabeled previous image, or `force_rebuild=true`) → `docker compose build` / `push` for those services only, labeling the new digest with `org.airstack.content-fingerprint`. + 5. Keyless `cosign sign` of every published image digest via GitHub OIDC. + 6. `cosign verify` against the workflow's certificate identity. - **Skip behavior:** if the merge commit on `main`/`develop` does not actually change `VERSION=`, the build job is skipped (the check-changes job sets `tag-changed=false`). +- **Docs-only VERSION bumps:** still required by `check-version-increment`, but publish should retag rather than rebuild once fingerprints are on the previous images. First publish after this feature lands (or `force_rebuild=true`) must rebuild to write the labels. ### 3. `deploy_docs_from_release.yaml` — versioned docs @@ -76,7 +78,7 @@ Three workflows in `.github/workflows/` interact with `VERSION`: - **Behavior:** runs `mike deploy --push --update-aliases latest`, publishing the docs site under the release tag and pointing the `latest` alias at it. - Companion workflows publish unversioned docs from `main` (default alias `main`) and `develop` (alias `develop`). -So the full release path is: bump `VERSION` → PR → merge to `main`/`develop` (rebuild + push + sign) → cut a GitHub Release matching that VERSION (versioned docs go live). +So the full release path is: bump `VERSION` → PR → merge to `main`/`develop` (retag unchanged images and/or rebuild changed ones + push + sign) → cut a GitHub Release matching that VERSION (versioned docs go live). ## Choosing the Bump Type diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index b415635fd..7c5195bef 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -12,6 +12,11 @@ on: required: false default: 'desktop,isaac-sim,ms-airsim' type: string + force_rebuild: + description: 'Force a full rebuild of every service (skip retag)' + required: false + default: false + type: boolean env: DEFAULT_PROFILES: 'desktop,isaac-sim,ms-airsim' @@ -21,6 +26,8 @@ jobs: runs-on: ubuntu-latest outputs: tag-changed: ${{ steps.check-changes.outputs.tag-changed }} + current-version: ${{ steps.check-changes.outputs.current-version }} + previous-version: ${{ steps.check-changes.outputs.previous-version }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -33,20 +40,22 @@ jobs: run: | # Get the current VERSION value CURRENT_TAG=$(grep "^VERSION=" .env | cut -d '=' -f2- | tr -d '"' | tr -d "'") - + # Get the previous VERSION value git show HEAD~1:.env > .env.prev 2>/dev/null || echo "" > .env.prev PREVIOUS_TAG=$(grep "^VERSION=" .env.prev | cut -d '=' -f2- | tr -d '"' | tr -d "'" || echo "") - + echo "Current tag: $CURRENT_TAG" echo "Previous tag: $PREVIOUS_TAG" - + echo "current-version=$CURRENT_TAG" >> "$GITHUB_OUTPUT" + echo "previous-version=$PREVIOUS_TAG" >> "$GITHUB_OUTPUT" + if [ "$CURRENT_TAG" != "$PREVIOUS_TAG" ] && [ -n "$CURRENT_TAG" ]; then echo "VERSION has changed from '$PREVIOUS_TAG' to '$CURRENT_TAG'" - echo "tag-changed=true" >> $GITHUB_OUTPUT + echo "tag-changed=true" >> "$GITHUB_OUTPUT" else echo "VERSION has not changed" - echo "tag-changed=false" >> $GITHUB_OUTPUT + echo "tag-changed=false" >> "$GITHUB_OUTPUT" fi docker-build: @@ -78,7 +87,6 @@ jobs: - name: Verify .env file and extract tag run: | - # Ensure .env file exists and is readable if [ ! -f .env ]; then echo "Error: .env file not found" exit 1 @@ -89,75 +97,161 @@ jobs: # Some compose files expect this file to exist, even if it is empty. mkdir -p simulation/isaac-sim/docker : > simulation/isaac-sim/docker/omni_pass.env - - # Display the current VERSION for debugging + DOCKER_TAG=$(grep "^VERSION=" .env | cut -d '=' -f2- | tr -d '"' | tr -d "'") echo "Building with VERSION: $DOCKER_TAG" - + if [ -z "$DOCKER_TAG" ]; then echo "Error: VERSION is empty" exit 1 fi - - name: Run Docker Compose Build + - name: Resolve compose profiles and previous VERSION + id: prep run: | - # Load environment variables and run docker compose build - set -a # Export all variables + set -a source .env - set +a # Stop exporting - - # Always override COMPOSE_PROFILES for all trigger types + set +a + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" else export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" fi - - docker compose build - - name: Run Docker Compose Push + PREVIOUS_VERSION="${{ needs.check-docker-tag-change.outputs.previous-version }}" + CURRENT_VERSION="${{ needs.check-docker-tag-change.outputs.current-version }}" + if [ -z "$CURRENT_VERSION" ]; then + CURRENT_VERSION=$(grep "^VERSION=" .env | cut -d '=' -f2- | tr -d '"' | tr -d "'") + fi + + FORCE_REBUILD=false + if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ github.event.inputs.force_rebuild }}" == "true" ]; then + FORCE_REBUILD=true + fi + + echo "COMPOSE_PROFILES=$COMPOSE_PROFILES" >> "$GITHUB_ENV" + echo "PREVIOUS_VERSION=$PREVIOUS_VERSION" >> "$GITHUB_ENV" + echo "CURRENT_VERSION=$CURRENT_VERSION" >> "$GITHUB_ENV" + echo "FORCE_REBUILD=$FORCE_REBUILD" >> "$GITHUB_ENV" + echo "profiles=$COMPOSE_PROFILES" >> "$GITHUB_OUTPUT" + echo "previous-version=$PREVIOUS_VERSION" >> "$GITHUB_OUTPUT" + echo "force-rebuild=$FORCE_REBUILD" >> "$GITHUB_OUTPUT" + + # Content-aware publish: retag previous versioned images when image inputs + # are unchanged; rebuild only services whose fingerprint differs (or when + # force_rebuild / unlabeled previous images force a cold build). + - name: Plan retag vs rebuild + id: plan run: | - # Load environment variables and run docker compose push - set -a # Export all variables + set -a source .env - set +a # Stop exporting - - # Always override COMPOSE_PROFILES for all trigger types - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" - else - export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" + set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" + + FORCE_ARGS=() + if [ "${{ env.FORCE_REBUILD }}" = "true" ]; then + FORCE_ARGS+=(--force-rebuild) fi - - docker compose push - # `docker compose push` only publishes each service's `image:` (the - # versioned tag). PR builds can never hit that tag as cache, because - # check-version-increment forces VERSION up on every PR — so they also - # cache_from a floating CACHE_TAG that only this workflow republishes. - # `docker compose build` already applied these via `build.tags`; they - # point at the digest just pushed, so Cosign's digest-based signature - # covers them too. - - name: Publish floating cache tags + python3 .github/workflows/scripts/docker_image_plan.py \ + --version "${{ env.CURRENT_VERSION }}" \ + --previous-version "${{ env.PREVIOUS_VERSION }}" \ + --profiles "${{ env.COMPOSE_PROFILES }}" \ + --plan-out docker-image-plan.json \ + --override-out docker-compose.fingerprint.yaml \ + "${FORCE_ARGS[@]}" + + echo "Plan summary:" + jq -r '.services | to_entries[] | " \(.key): \(.value.action) (\(.value.reason))"' docker-image-plan.json + + - name: Retag unchanged images run: | + set -euo pipefail + RETAG_COUNT=$(jq '[.services[] | select(.action=="retag")] | length' docker-image-plan.json) + echo "Services to retag: $RETAG_COUNT" + if [ "$RETAG_COUNT" -eq 0 ]; then + echo "Nothing to retag." + exit 0 + fi + + jq -c '.services | to_entries[] | select(.value.action=="retag") | .value' docker-image-plan.json \ + | while IFS= read -r row; do + IMAGE=$(echo "$row" | jq -r '.image') + PREV=$(echo "$row" | jq -r '.previous_image') + CACHE=$(echo "$row" | jq -r '.cache_tag // empty') + echo "Retagging $PREV → $IMAGE" + CREATE_ARGS=(--tag "$IMAGE") + if [ -n "$CACHE" ] && [ "$CACHE" != "null" ]; then + echo " also → $CACHE" + CREATE_ARGS+=(--tag "$CACHE") + fi + docker buildx imagetools create "${CREATE_ARGS[@]}" "$PREV" + done + + - name: Build changed images + run: | + set -euo pipefail set -a source .env set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" - else - export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" + BUILD_SERVICES=$(jq -r '[.services | to_entries[] | select(.value.action=="build") | .key] | join(" ")' docker-image-plan.json) + if [ -z "$BUILD_SERVICES" ]; then + echo "No services require a rebuild." + exit 0 fi - TAGS=$(docker compose config --format json \ - | jq -r --arg pfx ":${CACHE_TAG:-cache}_" \ - '.services[].build.tags // [] | .[] | select(contains($pfx))' \ - | sort -u) + echo "Building services: $BUILD_SERVICES" + # Override applies org.airstack.content-fingerprint build labels. + # shellcheck disable=SC2086 + docker compose \ + -f docker-compose.yaml \ + -f docker-compose.fingerprint.yaml \ + build $BUILD_SERVICES + + - name: Push rebuilt images + run: | + set -euo pipefail + set -a + source .env + set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" + + BUILD_SERVICES=$(jq -r '[.services | to_entries[] | select(.value.action=="build") | .key] | join(" ")' docker-image-plan.json) + if [ -z "$BUILD_SERVICES" ]; then + echo "No rebuilt images to push." + exit 0 + fi + + # shellcheck disable=SC2086 + docker compose \ + -f docker-compose.yaml \ + -f docker-compose.fingerprint.yaml \ + push $BUILD_SERVICES + + # `docker compose push` only publishes each service's `image:` (the + # versioned tag). Floating CACHE_TAG entries are also applied via + # build.tags on rebuild; retag already published them via imagetools. + - name: Publish floating cache tags for rebuilt images + run: | + set -euo pipefail + set -a + source .env + set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" + + TAGS=$(jq -r --arg pfx ":${CACHE_TAG:-cache}_" ' + .services | to_entries[] + | select(.value.action=="build") + | .value.cache_tag // empty + | select(length > 0 and contains($pfx)) + ' docker-image-plan.json | sort -u) if [ -z "$TAGS" ]; then - echo "No cache tags resolved from compose config; nothing to publish." - exit 1 + echo "No rebuilt cache tags to publish (retag path already set them, or no rebuilds)." + exit 0 fi for TAG in $TAGS; do @@ -165,23 +259,19 @@ jobs: docker push "$TAG" done - - name: Sign pushed images with Cosign (keyless) + - name: Sign published images with Cosign (keyless) env: COSIGN_YES: "true" run: | + set -euo pipefail set -a source .env set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" - else - export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" - fi - - IMAGES=$(docker compose config --images | sort -u) + IMAGES=$(jq -r '.services[].image' docker-image-plan.json | sort -u) if [ -z "$IMAGES" ]; then - echo "No images resolved from compose config; nothing to sign." + echo "No images resolved from plan; nothing to sign." exit 1 fi @@ -199,17 +289,13 @@ jobs: - name: Verify Cosign signatures run: | + set -euo pipefail set -a source .env set +a + export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - export COMPOSE_PROFILES="${{ github.event.inputs.compose_profiles || env.DEFAULT_PROFILES }}" - else - export COMPOSE_PROFILES="${{ env.DEFAULT_PROFILES }}" - fi - - IMAGES=$(docker compose config --images | sort -u) + IMAGES=$(jq -r '.services[].image' docker-image-plan.json | sort -u) for IMG in $IMAGES; do DIGEST=$(docker buildx imagetools inspect "$IMG" --format '{{.Manifest.Digest}}') REPO="${IMG%:*}" @@ -221,13 +307,15 @@ jobs: > /dev/null done - - name: Optional - Run Docker Compose Up (uncomment if needed) - run: | - # Uncomment the following lines if you also want to start the services - # set -a - # source .env - # set +a - # docker compose up -d + - name: Upload image plan artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: docker-image-plan-${{ github.run_id }} + path: | + docker-image-plan.json + docker-compose.fingerprint.yaml + if-no-files-found: ignore notify: needs: [check-docker-tag-change, docker-build] @@ -237,8 +325,8 @@ jobs: - name: Notify build and push result run: | if [ "${{ needs.docker-build.result }}" == "success" ]; then - echo "✅ Docker Compose build and push completed successfully" + echo "✅ Docker Compose build/retag and push completed successfully" else - echo "❌ Docker Compose build or push failed" + echo "❌ Docker Compose build/retag or push failed" exit 1 - fi \ No newline at end of file + fi diff --git a/.github/workflows/scripts/docker_image_plan.py b/.github/workflows/scripts/docker_image_plan.py new file mode 100755 index 000000000..b4963dece --- /dev/null +++ b/.github/workflows/scripts/docker_image_plan.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +"""Plan retag-vs-rebuild for AirStack docker-build.yml publishes. + +For each compose service with a ``build:`` section under the selected profiles, +compute a content fingerprint of its image inputs. If the previous versioned +image carries the same ``org.airstack.content-fingerprint`` label, the service +is marked ``retag``; otherwise ``build``. + +Also writes an ephemeral compose override that applies the fingerprint as a +build label on services that will be rebuilt. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +FINGERPRINT_LABEL = "org.airstack.content-fingerprint" + +# Explicit roots so broad compose ``context:`` dirs do not hash the whole tree. +# Keys are compose service names after ``docker compose config`` resolution. +SERVICE_FINGERPRINT_ROOTS: dict[str, list[str]] = { + "robot-desktop": [ + "robot/docker/Dockerfile.robot", + "robot/docker/docker-compose.yaml", + "robot/docker/robot-base-docker-compose.yaml", + "robot/docker/custom_rosdep.yaml", + "robot/docker/wait_for_px4.py", + "robot/docker/.bashrc", + "robot/docker/robot_name_map", + ], + "gcs": [ + "gcs/docker/Dockerfile.gcs", + "gcs/docker/docker-compose.yaml", + "gcs/docker/gcs-base-docker-compose.yaml", + "gcs/docker/.bashrc", + "gcs/docker/resources", + "gcs/docker/Foxglove", + ], + "isaac-sim": [ + "simulation/isaac-sim/docker/Dockerfile.isaac-ros", + "simulation/isaac-sim/docker/docker-compose.yaml", + "simulation/isaac-sim/docker/fastdds.xml", + "simulation/isaac-sim/docker/.bashrc", + "simulation/isaac-sim/docker/omniverse.toml", + ], + "ms-airsim": [ + "simulation/ms-airsim/docker/Dockerfile", + "simulation/ms-airsim/docker/docker-compose.yaml", + "simulation/ms-airsim/docker/entrypoint.sh", + ], +} + +# Extra roots when DOCKER_IMAGE_BUILD_MODE=prebuilt (workspace baked into image). +PREBUILT_EXTRA_ROOTS: dict[str, list[str]] = { + "robot-desktop": [ + "robot/ros_ws/src", + "common/ros_packages", + "common/fastdds.xml", + ], + "gcs": [ + "gcs/ros_ws", + "common/ros_packages", + ], +} + +# .env keys whose values affect image tags or layers (included in fingerprint). +ENV_FINGERPRINT_KEYS = ( + "DOCKER_IMAGE_BUILD_MODE", + "PROJECT_DOCKER_REGISTRY", + "PROJECT_NAME", + "CACHE_TAG", +) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def load_dotenv(path: Path) -> dict[str, str]: + env: dict[str, str] = {} + if not path.is_file(): + return env + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + value = value.strip().strip('"').strip("'") + env[key.strip()] = value + return env + + +def run(cmd: list[str], *, cwd: Path, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + cwd=cwd, + check=check, + text=True, + capture_output=True, + ) + + +def compose_config(root: Path, profiles: str, env: dict[str, str]) -> dict[str, Any]: + cmd_env = os.environ.copy() + cmd_env.update(env) + cmd_env["COMPOSE_PROFILES"] = profiles + proc = subprocess.run( + ["docker", "compose", "-f", "docker-compose.yaml", "config", "--format", "json"], + cwd=root, + env=cmd_env, + text=True, + capture_output=True, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + "docker compose config failed:\n" + f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + return json.loads(proc.stdout) + + +def git_ls_files(root: Path, pathspec: str) -> list[str]: + proc = run( + ["git", "ls-files", "-z", "--", pathspec], + cwd=root, + check=False, + ) + if proc.returncode != 0: + return [] + return [p for p in proc.stdout.split("\0") if p] + + +def collect_tracked_files(root: Path, roots: list[str]) -> list[str]: + files: set[str] = set() + for rel in roots: + path = root / rel + if path.is_file(): + files.add(rel) + continue + if path.is_dir(): + for tracked in git_ls_files(root, rel): + # Skip local secrets / generated pass files + if tracked.endswith("omni_pass.env"): + continue + if "/.dev/" in f"/{tracked}/" or tracked.endswith("/.dev"): + continue + files.add(tracked) + return sorted(files) + + +def file_sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def resolve_dockerfile(root: Path, service_cfg: dict[str, Any]) -> Path | None: + build = service_cfg.get("build") or {} + if not isinstance(build, dict): + return None + dockerfile = build.get("dockerfile") + context = build.get("context") or "." + if not dockerfile: + return None + # compose config usually resolves dockerfile to an absolute path + df = Path(dockerfile) + if df.is_absolute(): + return df + return (Path(context) / df).resolve() if Path(context).is_absolute() else (root / context / df).resolve() + + +def find_dockerignore(dockerfile: Path, context: Path) -> Path | None: + for candidate in ( + context / ".dockerignore", + dockerfile.parent / ".dockerignore", + ): + if candidate.is_file(): + return candidate + return None + + +def previous_image_ref(image: str, version: str, previous_version: str) -> str | None: + if not previous_version or not version or version == previous_version: + return None + # Tags look like ...:v{VERSION}_suffix — replace only the version segment. + needle = f":v{version}_" + if needle not in image: + # Fallback: replace first occurrence of the bare version in the tag. + tag_part = image.rsplit(":", 1) + if len(tag_part) != 2 or version not in tag_part[1]: + return None + return f"{tag_part[0]}:{tag_part[1].replace(version, previous_version, 1)}" + return image.replace(needle, f":v{previous_version}_", 1) + + +def cache_tag_from_build(build: dict[str, Any], cache_tag: str) -> str | None: + tags = build.get("tags") or [] + pfx = f":{cache_tag}_" + for tag in tags: + if isinstance(tag, str) and pfx in tag: + return tag + return None + + +def inspect_fingerprint_label(image: str) -> str | None: + """Return the fingerprint label from a registry image, or None if unavailable.""" + proc = subprocess.run( + [ + "docker", + "buildx", + "imagetools", + "inspect", + image, + "--format", + "{{json .}}", + ], + text=True, + capture_output=True, + check=False, + ) + if proc.returncode != 0: + return None + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + return None + + # buildx JSON shape varies by version; hunt for Labels in common places. + def walk(obj: Any) -> str | None: + if isinstance(obj, dict): + labels = obj.get("Labels") or obj.get("labels") + if isinstance(labels, dict) and FINGERPRINT_LABEL in labels: + return labels[FINGERPRINT_LABEL] + for v in obj.values(): + found = walk(v) + if found: + return found + elif isinstance(obj, list): + for item in obj: + found = walk(item) + if found: + return found + return None + + return walk(data) + + +def compute_fingerprint( + root: Path, + service_name: str, + service_cfg: dict[str, Any], + env: dict[str, str], +) -> str: + build = service_cfg.get("build") or {} + roots = list(SERVICE_FINGERPRINT_ROOTS.get(service_name, [])) + + dockerfile = resolve_dockerfile(root, service_cfg) + if dockerfile and dockerfile.is_file(): + try: + rel = str(dockerfile.relative_to(root)) + except ValueError: + rel = str(dockerfile) + if rel not in roots: + roots.insert(0, rel) + + mode = env.get("DOCKER_IMAGE_BUILD_MODE", "dev") + if mode == "prebuilt": + roots.extend(PREBUILT_EXTRA_ROOTS.get(service_name, [])) + + # If service has no map entry, fall back to dockerfile directory. + if not roots and dockerfile is not None: + try: + roots = [str(dockerfile.parent.relative_to(root))] + except ValueError: + roots = [] + + tracked = collect_tracked_files(root, roots) + + h = hashlib.sha256() + h.update(f"service:{service_name}\n".encode()) + h.update(f"DOCKER_IMAGE_BUILD_MODE:{mode}\n".encode()) + + for key in ENV_FINGERPRINT_KEYS: + h.update(f"env:{key}={env.get(key, '')}\n".encode()) + + args = build.get("args") or {} + if isinstance(args, dict): + for k in sorted(args): + h.update(f"arg:{k}={args[k]}\n".encode()) + + context = build.get("context") + if context: + h.update(f"context:{context}\n".encode()) + ctx_path = Path(context) if Path(context).is_absolute() else root / context + dockerignore = find_dockerignore(dockerfile, ctx_path) if dockerfile else None + if dockerignore and dockerignore.is_file(): + h.update(f"dockerignore:{dockerignore.name}\n".encode()) + h.update(dockerignore.read_bytes()) + h.update(b"\n") + + for rel in tracked: + path = root / rel + if not path.is_file(): + continue + h.update(f"file:{rel}\n".encode()) + h.update(file_sha256(path).encode()) + h.update(b"\n") + + return h.hexdigest() + + +def write_override(path: Path, build_services: dict[str, str]) -> None: + """Write compose override that sets build.labels fingerprint for rebuilds.""" + lines = [ + "# Generated by docker_image_plan.py — do not commit.", + "services:", + ] + if not build_services: + # Valid empty mapping; compose merge ignores it when nothing rebuilds. + lines[-1] = "services: {}" + else: + for name, fingerprint in sorted(build_services.items()): + lines.append(f" {name}:") + lines.append(" build:") + lines.append(" labels:") + lines.append(f" {FINGERPRINT_LABEL}: \"{fingerprint}\"") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def build_plan( + root: Path, + *, + version: str, + previous_version: str, + profiles: str, + force_rebuild: bool, + env: dict[str, str], +) -> dict[str, Any]: + config = compose_config(root, profiles, env) + services_out: dict[str, Any] = {} + cache_tag = env.get("CACHE_TAG") or "cache" + + for name, cfg in (config.get("services") or {}).items(): + if not isinstance(cfg, dict): + continue + build = cfg.get("build") + if not isinstance(build, dict) or not build: + continue + + image = cfg.get("image") + if not isinstance(image, str) or not image: + continue + + fingerprint = compute_fingerprint(root, name, cfg, env) + prev_image = previous_image_ref(image, version, previous_version) + cache_image = cache_tag_from_build(build, cache_tag) + + action = "build" + reason = "force_rebuild" if force_rebuild else "default_build" + prev_fp = None + if force_rebuild: + action = "build" + reason = "force_rebuild" + elif not prev_image: + action = "build" + reason = "no_previous_image" + else: + prev_fp = inspect_fingerprint_label(prev_image) + if prev_fp is None: + action = "build" + reason = "previous_missing_or_unlabeled" + elif prev_fp == fingerprint: + action = "retag" + reason = "fingerprint_match" + else: + action = "build" + reason = "fingerprint_mismatch" + + services_out[name] = { + "image": image, + "previous_image": prev_image, + "cache_tag": cache_image, + "fingerprint": fingerprint, + "previous_fingerprint": prev_fp, + "action": action, + "reason": reason, + } + + return { + "previous_version": previous_version, + "version": version, + "profiles": profiles, + "force_rebuild": force_rebuild, + "label": FINGERPRINT_LABEL, + "services": services_out, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=None, help="Repo root (default: AirStack/)") + parser.add_argument("--version", default="", help="Current VERSION (default: from .env)") + parser.add_argument("--previous-version", default="", help="Previous VERSION for retag source") + parser.add_argument( + "--profiles", + default="", + help="Compose profiles (default: COMPOSE_PROFILES or desktop,isaac-sim,ms-airsim)", + ) + parser.add_argument( + "--force-rebuild", + action="store_true", + help="Mark every service as build", + ) + parser.add_argument( + "--plan-out", + type=Path, + default=Path("docker-image-plan.json"), + help="Where to write the plan JSON", + ) + parser.add_argument( + "--override-out", + type=Path, + default=Path("docker-compose.fingerprint.yaml"), + help="Compose override with build labels for rebuild services", + ) + parser.add_argument( + "--github-output", + type=Path, + default=None, + help="Optional path to append GITHUB_OUTPUT keys", + ) + args = parser.parse_args(argv) + + root = (args.root or repo_root()).resolve() + env = load_dotenv(root / ".env") + # Prefer process env overlays (CI exports .env via set -a). + for key in ( + "VERSION", + "DOCKER_IMAGE_BUILD_MODE", + "PROJECT_DOCKER_REGISTRY", + "PROJECT_NAME", + "CACHE_TAG", + "COMPOSE_PROFILES", + ): + if os.environ.get(key): + env[key] = os.environ[key] + + version = args.version or env.get("VERSION") or "" + if not version: + print("ERROR: VERSION is empty", file=sys.stderr) + return 1 + + previous_version = args.previous_version + profiles = ( + args.profiles + or os.environ.get("COMPOSE_PROFILES") + or env.get("COMPOSE_PROFILES") + or "desktop,isaac-sim,ms-airsim" + ) + + plan = build_plan( + root, + version=version, + previous_version=previous_version, + profiles=profiles, + force_rebuild=args.force_rebuild, + env=env, + ) + + build_services = { + name: svc["fingerprint"] + for name, svc in plan["services"].items() + if svc["action"] == "build" + } + retag_services = [name for name, svc in plan["services"].items() if svc["action"] == "retag"] + + args.plan_out = args.plan_out if args.plan_out.is_absolute() else root / args.plan_out + args.override_out = ( + args.override_out if args.override_out.is_absolute() else root / args.override_out + ) + args.plan_out.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_override(args.override_out, build_services) + + print(f"Wrote plan → {args.plan_out}") + print(f"Wrote override → {args.override_out}") + for name, svc in sorted(plan["services"].items()): + print( + f" {name}: action={svc['action']} reason={svc['reason']} " + f"fp={svc['fingerprint'][:12]}…" + ) + + build_list = " ".join(sorted(build_services)) + retag_list = " ".join(sorted(retag_services)) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as fh: + fh.write(f"plan_path={args.plan_out}\n") + fh.write(f"override_path={args.override_out}\n") + fh.write(f"build_services={build_list}\n") + fh.write(f"retag_services={retag_list}\n") + fh.write(f"build_count={len(build_services)}\n") + fh.write(f"retag_count={len(retag_services)}\n") + else: + # Also support GITHUB_OUTPUT env when set by Actions. + gh_out = os.environ.get("GITHUB_OUTPUT") + if gh_out: + with open(gh_out, "a", encoding="utf-8") as fh: + fh.write(f"plan_path={args.plan_out}\n") + fh.write(f"override_path={args.override_out}\n") + fh.write(f"build_services={build_list}\n") + fh.write(f"retag_services={retag_list}\n") + fh.write(f"build_count={len(build_services)}\n") + fh.write(f"retag_count={len(retag_services)}\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitignore b/.gitignore index 4868b5c74..511ff44c0 100644 --- a/.gitignore +++ b/.gitignore @@ -99,5 +99,10 @@ common/rayfronts/ # Docker build cache (root-owned subdirs cause permission warnings on `git add`) robot/docker/cache/ + +# Ephemeral outputs from docker_image_plan.py (docker-build.yml) +docker-image-plan.json +docker-compose.fingerprint.yaml + .DS_Store gcs/.DS_Store diff --git a/AGENTS.md b/AGENTS.md index e19877727..856608098 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -287,6 +287,8 @@ failed to solve: ... mount source: "overlay", target: ".../buildkit/containerd-o **Docker layer cache is a floating tag, not the versioned one.** Every compose service lists two `cache_from` entries: the versioned image (`airstack:v${VERSION}_`) and a floating one (`airstack:${CACHE_TAG:-cache}_`). Only the floating tag can ever hit on a PR — `check-version-increment` forces `VERSION` up on every PR, so the versioned tag it builds under has by definition never been pushed. Reading and writing are separate switches: `AIRSTACK_REGISTRY_CACHE=1` (set by `system-tests.yml`) pulls and builds with `BUILDKIT_INLINE_CACHE=1`, while `AIRSTACK_REGISTRY_CACHE_PUSH=1` (set only by `docker-build.yml` on main/develop) also publishes both tags. PR runs stay read-only so an unmerged branch can't poison the shared cache or publish an unreleased version. If you add a service with a `build:` section, give it both entries or its builds will always be cold. +**Publish retags when image inputs are unchanged.** `docker-build.yml` runs [`.github/workflows/scripts/docker_image_plan.py`](.github/workflows/scripts/docker_image_plan.py) on VERSION bumps: each service gets a content fingerprint (`org.airstack.content-fingerprint`). If the previous versioned image already has that label, the job registry-retags (`imagetools create`) instead of rebuilding; only changed services rebuild (and refresh `cache_*`). Use `workflow_dispatch` with `force_rebuild=true` to rebuild everything. PR `build_docker` tests still perform real builds. + **Setup, debugging a failed job, and exec-into-worker procedures:** [`.github/orchestrator/README.md`](.github/orchestrator/README.md) (also exposed as [`tests/ci-cd-orchestrator.md`](tests/ci-cd-orchestrator.md) symlink for the docs site). ## Documentation Requirements diff --git a/docs/development/intermediate/testing/ci_cd.md b/docs/development/intermediate/testing/ci_cd.md index 175b4c8e1..40a16af82 100644 --- a/docs/development/intermediate/testing/ci_cd.md +++ b/docs/development/intermediate/testing/ci_cd.md @@ -274,6 +274,34 @@ neither poison the shared cache for everyone else nor publish an unreleased `VERSION` tag. Override the tag name with `CACHE_TAG` (default `cache`) to keep an experimental cache line separate. +### Publish path: retag when image inputs are unchanged + +`check-version-increment` forces every PR to raise `VERSION`, including +docs-only changes. On `main`/`develop`, that would otherwise mean a full +multi-hour rebuild of every image for a no-op Docker change. + +[`docker-build.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/docker-build.yml) +therefore plans per service before building: + +1. [`.github/workflows/scripts/docker_image_plan.py`](https://github.com/castacks/AirStack/blob/main/.github/workflows/scripts/docker_image_plan.py) + hashes each service’s Dockerfile, compose-related files, build args, and + tracked fingerprint roots into `org.airstack.content-fingerprint`. +2. It inspects the **previous** versioned image’s label (from `HEAD~1`’s + `VERSION=`). +3. **Match** → registry-side retag with + `docker buildx imagetools create` (new `v${VERSION}_…` tag and floating + `cache_*` tag, same digest — no rebuild). +4. **Mismatch / missing / unlabeled / `force_rebuild`** → `docker compose build` + for that service only, with the fingerprint applied as a build label via an + ephemeral `docker-compose.fingerprint.yaml` override. + +PR `system-tests` / `build_docker` are unchanged: they still run real builds so +Dockerfiles keep being proven. Floating `cache_*` remains the layer-cache seed +for those rebuilds. + +Manual dispatch accepts `force_rebuild=true` to rebuild and relabel everything +(useful the first time after this lands, or to refresh `cache_*` from scratch). + --- ## What the pipeline tests, and what that catches @@ -435,20 +463,24 @@ flowchart LR pr["PR merged to main or develop"] --> chk{".env VERSION changed?"} chk -- no --> stop["No build"] chk -- yes --> pod["Ephemeral OSMO pod"] - pod --> build["docker compose build"] - build --> push["docker compose push"] - push --> sign["cosign sign — keyless, GitHub OIDC"] + pod --> plan["Per-service fingerprint plan"] + plan --> retag["imagetools retag unchanged"] + plan --> build["compose build changed only"] + retag --> sign["cosign sign — keyless, GitHub OIDC"] + build --> sign sign --> verify["cosign verify against the workflow identity"] ``` Signing is keyless via GitHub's OIDC token, and the same job immediately -verifies each pushed digest against the expected certificate identity, so a -published image that was not built by this workflow fails the check. +verifies each published digest against the expected certificate identity, so a +published image that was not built by this workflow fails the check. Retagged +images keep the previous digest (and therefore an existing signature still +covers that digest; the job re-signs the same digest under the new tags’ refs). | Workflow | Runner | Purpose | |---|---|---| | `system-tests.yml` | Ephemeral OSMO GPU pod | Full test suite + metrics report | -| `docker-build.yml` | Ephemeral OSMO GPU pod | Build, push, and sign all compose images | +| `docker-build.yml` | Ephemeral OSMO GPU pod | Retag or rebuild, push, and sign compose images | | `check-version-increment.yml` | `ubuntu-latest` | Semver gate on `.env` `VERSION=` | | `deploy_docs_from_*.yaml` | `ubuntu-latest` | Versioned MkDocs publish via `mike` | From b093327b69f70c07d84085802daa3a11ecbbb58f Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Fri, 7 Aug 2026 17:29:13 -0400 Subject: [PATCH 17/27] fix(ci): parse quoted .env values before inline comments docker_image_plan was feeding NUM_ROBOTS with a trailing comment into compose config, which broke strconv.Atoi for deploy.replicas. Co-authored-by: Cursor --- .../workflows/scripts/docker_image_plan.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/docker_image_plan.py b/.github/workflows/scripts/docker_image_plan.py index b4963dece..dcaf1a7ae 100755 --- a/.github/workflows/scripts/docker_image_plan.py +++ b/.github/workflows/scripts/docker_image_plan.py @@ -83,6 +83,23 @@ def repo_root() -> Path: return Path(__file__).resolve().parents[3] +def parse_env_value(raw: str) -> str: + """Parse a .env value, honoring quotes and stripping trailing comments.""" + raw = raw.strip() + if not raw: + return "" + if raw[0] in "\"'": + quote = raw[0] + end = raw.find(quote, 1) + if end != -1: + return raw[1:end] + return raw[1:] + # Unquoted: drop an inline ` # comment` (space-hash) or a leading `#`. + if " #" in raw: + raw = raw.split(" #", 1)[0].rstrip() + return raw.strip().strip('"').strip("'") + + def load_dotenv(path: Path) -> dict[str, str]: env: dict[str, str] = {} if not path.is_file(): @@ -92,8 +109,7 @@ def load_dotenv(path: Path) -> dict[str, str]: if not line or line.startswith("#") or "=" not in line: continue key, _, value = line.partition("=") - value = value.strip().strip('"').strip("'") - env[key.strip()] = value + env[key.strip()] = parse_env_value(value) return env From 56a60c7cc6c159c9e188e19b9073d6bffb9700c1 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Fri, 7 Aug 2026 21:06:43 -0400 Subject: [PATCH 18/27] ci(docker-build): build/push services sequentially Publish successful images even when a sibling (e.g. isaac-sim) fails, and still cosign whatever was retagged or pushed in the same run. Co-authored-by: Cursor --- .github/workflows/docker-build.yml | 139 ++++++++++++++++------------- 1 file changed, 78 insertions(+), 61 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 7c5195bef..403864781 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -189,77 +189,66 @@ jobs: docker buildx imagetools create "${CREATE_ARGS[@]}" "$PREV" done - - name: Build changed images + # Build/push one service at a time so a single Dockerfile failure (e.g. + # isaac-sim PX4 apt) does not discard successful siblings before push. + - name: Build and push changed images + id: build_push run: | - set -euo pipefail + set -uo pipefail set -a source .env set +a export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - BUILD_SERVICES=$(jq -r '[.services | to_entries[] | select(.value.action=="build") | .key] | join(" ")' docker-image-plan.json) - if [ -z "$BUILD_SERVICES" ]; then + mapfile -t BUILD_SERVICES < <(jq -r '.services | to_entries[] | select(.value.action=="build") | .key' docker-image-plan.json | sort) + if [ "${#BUILD_SERVICES[@]}" -eq 0 ]; then echo "No services require a rebuild." + echo "built_services=" >> "$GITHUB_OUTPUT" exit 0 fi - echo "Building services: $BUILD_SERVICES" - # Override applies org.airstack.content-fingerprint build labels. - # shellcheck disable=SC2086 - docker compose \ - -f docker-compose.yaml \ - -f docker-compose.fingerprint.yaml \ - build $BUILD_SERVICES - - - name: Push rebuilt images - run: | - set -euo pipefail - set -a - source .env - set +a - export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - - BUILD_SERVICES=$(jq -r '[.services | to_entries[] | select(.value.action=="build") | .key] | join(" ")' docker-image-plan.json) - if [ -z "$BUILD_SERVICES" ]; then - echo "No rebuilt images to push." - exit 0 - fi - - # shellcheck disable=SC2086 - docker compose \ - -f docker-compose.yaml \ - -f docker-compose.fingerprint.yaml \ - push $BUILD_SERVICES - - # `docker compose push` only publishes each service's `image:` (the - # versioned tag). Floating CACHE_TAG entries are also applied via - # build.tags on rebuild; retag already published them via imagetools. - - name: Publish floating cache tags for rebuilt images - run: | - set -euo pipefail - set -a - source .env - set +a - export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - - TAGS=$(jq -r --arg pfx ":${CACHE_TAG:-cache}_" ' - .services | to_entries[] - | select(.value.action=="build") - | .value.cache_tag // empty - | select(length > 0 and contains($pfx)) - ' docker-image-plan.json | sort -u) + echo "Building services sequentially: ${BUILD_SERVICES[*]}" + FAILED=() + BUILT=() + for SVC in "${BUILD_SERVICES[@]}"; do + echo "::group::Build $SVC" + if docker compose \ + -f docker-compose.yaml \ + -f docker-compose.fingerprint.yaml \ + build "$SVC"; then + echo "::endgroup::" + echo "::group::Push $SVC" + if docker compose \ + -f docker-compose.yaml \ + -f docker-compose.fingerprint.yaml \ + push "$SVC"; then + CACHE=$(jq -r --arg s "$SVC" '.services[$s].cache_tag // empty' docker-image-plan.json) + if [ -n "$CACHE" ]; then + echo "Pushing cache tag $CACHE" + docker push "$CACHE" || echo "::warning::Failed to push cache tag $CACHE" + fi + BUILT+=("$SVC") + else + echo "::error::Push failed for $SVC" + FAILED+=("$SVC") + fi + echo "::endgroup::" + else + echo "::endgroup::" + echo "::error::Build failed for $SVC" + FAILED+=("$SVC") + fi + done - if [ -z "$TAGS" ]; then - echo "No rebuilt cache tags to publish (retag path already set them, or no rebuilds)." - exit 0 + echo "built_services=${BUILT[*]}" >> "$GITHUB_OUTPUT" + if [ "${#FAILED[@]}" -gt 0 ]; then + echo "Failed services: ${FAILED[*]}" + exit 1 fi - for TAG in $TAGS; do - echo "Pushing cache tag $TAG" - docker push "$TAG" - done - + # Sign whatever was published even if a sibling service build failed. - name: Sign published images with Cosign (keyless) + if: always() && steps.plan.outcome == 'success' env: COSIGN_YES: "true" run: | @@ -269,10 +258,21 @@ jobs: set +a export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - IMAGES=$(jq -r '.services[].image' docker-image-plan.json | sort -u) + # Retagged services are always published; rebuilt ones only if push succeeded. + BUILT_CSV="${{ steps.build_push.outputs.built_services }}" + IMAGES=$( + { + jq -r '.services | to_entries[] | select(.value.action=="retag") | .value.image' docker-image-plan.json + if [ -n "$BUILT_CSV" ]; then + for SVC in $BUILT_CSV; do + jq -r --arg s "$SVC" '.services[$s].image // empty' docker-image-plan.json + done + fi + } | awk 'NF' | sort -u + ) if [ -z "$IMAGES" ]; then - echo "No images resolved from plan; nothing to sign." - exit 1 + echo "No published images to sign." + exit 0 fi for IMG in $IMAGES; do @@ -288,6 +288,7 @@ jobs: done - name: Verify Cosign signatures + if: always() && steps.plan.outcome == 'success' run: | set -euo pipefail set -a @@ -295,7 +296,17 @@ jobs: set +a export COMPOSE_PROFILES="${{ env.COMPOSE_PROFILES }}" - IMAGES=$(jq -r '.services[].image' docker-image-plan.json | sort -u) + BUILT_CSV="${{ steps.build_push.outputs.built_services }}" + IMAGES=$( + { + jq -r '.services | to_entries[] | select(.value.action=="retag") | .value.image' docker-image-plan.json + if [ -n "$BUILT_CSV" ]; then + for SVC in $BUILT_CSV; do + jq -r --arg s "$SVC" '.services[$s].image // empty' docker-image-plan.json + done + fi + } | awk 'NF' | sort -u + ) for IMG in $IMAGES; do DIGEST=$(docker buildx imagetools inspect "$IMG" --format '{{.Manifest.Digest}}') REPO="${IMG%:*}" @@ -307,6 +318,12 @@ jobs: > /dev/null done + - name: Fail job if any service build/push failed + if: always() && steps.build_push.outcome == 'failure' + run: | + echo "One or more services failed to build or push (see Build and push changed images)." + exit 1 + - name: Upload image plan artifact if: always() uses: actions/upload-artifact@v4 From 624dec798ead19b218792ee03309b13a3a402b12 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Sat, 8 Aug 2026 01:07:16 -0400 Subject: [PATCH 19/27] chore: bump VERSION to 0.19.0-alpha.8 for retag validation Seeded gcs/ms-airsim/robot images carry content-fingerprint labels; this bump should registry-retag those digests without rebuilding. Co-authored-by: Cursor --- .env | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 85280be19..00cd4c639 100644 --- a/.env +++ b/.env @@ -12,7 +12,7 @@ PROJECT_NAME="airstack" # If you've run ./airstack.sh setup, then this will auto-generate from the git commit hash every time a change is made # to a Dockerfile or docker-compose.yaml file. Otherwise this can also be set explicitly to make a release version. # auto-generated from git commit hash -VERSION="0.19.0-alpha.7" +VERSION="0.19.0-alpha.8" # Choose "dev" or "prebuilt". "dev" is for mounted code that must be built live. "prebuilt" is for built ros_ws baked into the image DOCKER_IMAGE_BUILD_MODE="dev" # Where to push and pull images from. Can replace with your docker hub username if using docker hub. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bde19e49..ebc4e10ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `docker-build.yml` retags unchanged images on VERSION bumps (content fingerprint) instead of always rebuilding; floating `cache_*` tags still seed PR layer cache - `robot-l4t` compose service knobs are now env-overridable (`AUTONOMY_ROLE`, `FCU_URL`, and the rosbag path via `BAG_STORAGE_PATH`); `FCU_URL` unquoted so the literal serial path reaches MAVROS - `zed-l4t` image: ZED SDK 4.2 → 5.2 with the coupled ROS deps (`zed_msgs` 5.2.1, `point_cloud_transport(_plugins)` 4.x, add `backward_ros`) From d18efe976263f172cf03b312fbd7c86d541207d9 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Tue, 11 Aug 2026 15:56:04 -0400 Subject: [PATCH 20/27] fix(ci): unblock isaac-sim PX4 apt and robot colcon pytest Isaac's PX4 ubuntu.sh fails dpkg configure on the NVIDIA base; pre-fix ca-certificates, drop software-properties-common, and skip NuttX/Gazebo like ms-airsim. Pin pytest<8.1 and disable launch_testing for colcon unit tests so ROS Jazzy's outdated pytest hook no longer aborts CI. Co-authored-by: Cursor --- .env | 2 +- CHANGELOG.md | 2 ++ robot/docker/Dockerfile.robot | 6 ++++++ .../isaac-sim/docker/Dockerfile.isaac-ros | 20 ++++++++++++++++--- tests/colcon_unit_test_packages.yaml | 4 +++- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 00cd4c639..c9527e1f8 100644 --- a/.env +++ b/.env @@ -12,7 +12,7 @@ PROJECT_NAME="airstack" # If you've run ./airstack.sh setup, then this will auto-generate from the git commit hash every time a change is made # to a Dockerfile or docker-compose.yaml file. Otherwise this can also be set explicitly to make a release version. # auto-generated from git commit hash -VERSION="0.19.0-alpha.8" +VERSION="0.19.0-alpha.9" # Choose "dev" or "prebuilt". "dev" is for mounted code that must be built live. "prebuilt" is for built ros_ws baked into the image DOCKER_IMAGE_BUILD_MODE="dev" # Where to push and pull images from. Can replace with your docker hub username if using docker hub. diff --git a/CHANGELOG.md b/CHANGELOG.md index ebc4e10ab..e64a82978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Isaac Sim image: PX4 `ubuntu.sh` no longer fails dpkg configure on the NVIDIA base (`ca-certificates` / `software-properties-common`); use `--no-nuttx --no-sim-tools` like ms-airsim +- Robot image: pin `pytest<8.1` and disable `launch_testing` for colcon unit tests so ROS Jazzy's outdated pytest hook does not abort `colcon test` - Robot name resolution now honors a pre-set `ROBOT_NAME` (e.g. injected via docker compose) instead of always overriding it from the container/hostname mapping (`robot/docker/.bashrc`) - Robot name-map catch-all fallback now maps to `unknown_robot` (valid ROS namespace token) instead of `unknown-robot` (`default_robot_name_map.yaml`) - l4t robot image: replace dustynv's `/ros_entrypoint.sh` with a passthrough so its prebuilt source-ROS libs (older `fastcdr`) no longer shadow the apt Jazzy runtime and crash apt-built nodes like MAVROS diff --git a/robot/docker/Dockerfile.robot b/robot/docker/Dockerfile.robot index 92fd116fe..b97029f06 100644 --- a/robot/docker/Dockerfile.robot +++ b/robot/docker/Dockerfile.robot @@ -167,6 +167,12 @@ RUN pip3 install --break-system-packages --ignore-installed \ kornia \ typeguard==2.13.3 +# Keep pytest < 8.1. ROS Jazzy launch_testing still implements +# pytest_pycollect_makemodule(path=...), which pluggy rejects after pytest 8.1 +# removed the py.path hook argument (PluginValidationError on colcon test). +RUN python3 -m pip install --no-cache-dir --break-system-packages \ + "pytest>=7.4,<8.1" + # Install MACVO Python dependencies (skipped if SKIP_MACVO=true) RUN if [ "${SKIP_MACVO}" != "true" ]; then \ pip3 install --break-system-packages \ diff --git a/simulation/isaac-sim/docker/Dockerfile.isaac-ros b/simulation/isaac-sim/docker/Dockerfile.isaac-ros index 0dca11fb7..69bbd8117 100644 --- a/simulation/isaac-sim/docker/Dockerfile.isaac-ros +++ b/simulation/isaac-sim/docker/Dockerfile.isaac-ros @@ -154,9 +154,23 @@ RUN sed -i \ 's|param set-default IMU_INTEG_RATE 250|param set-default IMU_INTEG_RATE ${PX4_IMU_INTEG_RATE:-250}|' \ /isaac-sim/PX4-Autopilot/ROMFS/px4fmu_common/init.d-posix/px4-rc.simulator -# install px4 dependencies and build -RUN cd PX4-Autopilot && \ - DEBIAN_FRONTEND=noninteractive ./Tools/setup/ubuntu.sh +# Install PX4 host deps and build SITL. +# Match ms-airsim: skip NuttX + Gazebo — Isaac Sim is the simulator, and those +# toolchains are heavy. PX4's ubuntu.sh still apt-installs +# software-properties-common whenever /.dockerenv is present; on the nvcr.io +# Isaac base that package's configure step races with a half-configured +# ca-certificates/launchpadlib chain and fails with dpkg exit 100. Reconfigure +# ca-certificates first and drop software-properties-common from the script +# (add-apt-repository is unused on the --no-nuttx/--no-sim-tools path). +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates && \ + update-ca-certificates && \ + dpkg --configure -a || true && \ + sed -i '/software-properties-common/d' PX4-Autopilot/Tools/setup/ubuntu.sh && \ + cd PX4-Autopilot && \ + DEBIAN_FRONTEND=noninteractive ./Tools/setup/ubuntu.sh --no-nuttx --no-sim-tools && \ + rm -rf /var/lib/apt/lists/* + # build px4 RUN cd PX4-Autopilot && \ make px4_sitl diff --git a/tests/colcon_unit_test_packages.yaml b/tests/colcon_unit_test_packages.yaml index 5e0bd0ebb..4a64dd00d 100644 --- a/tests/colcon_unit_test_packages.yaml +++ b/tests/colcon_unit_test_packages.yaml @@ -10,4 +10,6 @@ robot: - natnet_ros2 - lidar_point_cloud_filter # Skips ament_copyright / flake8 / pep257 on Python packages; run linters separately. - pytest_args: "-m not linter" + # Disable launch_testing: these packages don't use it, and ROS Jazzy's plugin + # still declares the removed pytest_pycollect_makemodule(path=...) hook arg. + pytest_args: "-m not linter -p no:launch_testing" From d353eec1e0e683fe907b4c19d3f1885e2198a603 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Tue, 11 Aug 2026 18:37:07 -0400 Subject: [PATCH 21/27] fix(ci): pass colcon --pytest-args as separate tokens A single quoted blob made pytest treat "-p no:launch_testing" as part of the -m expression, which broke lidar_point_cloud_filter colcon tests. Co-authored-by: Cursor --- tests/colcon_unit_test_packages.yaml | 4 +++- tests/conftest.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/colcon_unit_test_packages.yaml b/tests/colcon_unit_test_packages.yaml index 4a64dd00d..7b4e68c08 100644 --- a/tests/colcon_unit_test_packages.yaml +++ b/tests/colcon_unit_test_packages.yaml @@ -12,4 +12,6 @@ robot: # Skips ament_copyright / flake8 / pep257 on Python packages; run linters separately. # Disable launch_testing: these packages don't use it, and ROS Jazzy's plugin # still declares the removed pytest_pycollect_makemodule(path=...) hook arg. - pytest_args: "-m not linter -p no:launch_testing" + # Tokens are split and passed as separate --pytest-args (see conftest). + # Quote the mark expression so "not linter" stays one argv after shlex.split. + pytest_args: '-m "not linter" -p no:launch_testing' diff --git a/tests/conftest.py b/tests/conftest.py index 2a51c569a..91f66ff44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,7 +81,11 @@ def colcon_test_robot_command(workspace="robot"): "--event-handlers console_direct+ --return-code-on-test-failure" ) if pytest_args: - cmd += f' --pytest-args "{pytest_args}"' + # One --pytest-args per token so flags like -p are not swallowed into + # the -m expression (colcon forwards a single quoted blob as one argv). + cmd += "".join( + f" --pytest-args {shlex.quote(a)}" for a in shlex.split(pytest_args) + ) return cmd # Unit tests live co-located with their ROS 2 packages in robot/ros_ws/src/. # Thin proxy files under tests/robot/ re-export those tests so that From 382202b7ad44bea17832562088bd07d887854ecd Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Tue, 11 Aug 2026 21:28:24 -0400 Subject: [PATCH 22/27] fix(ci): quote colcon pytest args through bash -ic Nested single quotes around 'not linter' terminated the outer bash -ic string early, so pytest saw 'not' as a path. Use shlex.quote for the whole command and list-form pytest_args in the YAML. Co-authored-by: Cursor --- tests/colcon_unit_test_packages.yaml | 9 ++++++--- tests/conftest.py | 16 ++++++++++++---- tests/system/test_build_packages.py | 5 ++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/colcon_unit_test_packages.yaml b/tests/colcon_unit_test_packages.yaml index 7b4e68c08..ff1a039fe 100644 --- a/tests/colcon_unit_test_packages.yaml +++ b/tests/colcon_unit_test_packages.yaml @@ -12,6 +12,9 @@ robot: # Skips ament_copyright / flake8 / pep257 on Python packages; run linters separately. # Disable launch_testing: these packages don't use it, and ROS Jazzy's plugin # still declares the removed pytest_pycollect_makemodule(path=...) hook arg. - # Tokens are split and passed as separate --pytest-args (see conftest). - # Quote the mark expression so "not linter" stays one argv after shlex.split. - pytest_args: '-m "not linter" -p no:launch_testing' + # List form: each entry is one argv token forwarded via --pytest-args (see conftest). + pytest_args: + - -m + - not linter + - -p + - no:launch_testing diff --git a/tests/conftest.py b/tests/conftest.py index 91f66ff44..ed7a09ff3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,7 +69,17 @@ def load_colcon_unit_test_config(workspace="robot"): raise ValueError( f"'{workspace}.packages' is empty in {COLCON_UNIT_TEST_PACKAGES_YAML.name}" ) - return packages, cfg.get("pytest_args", "") + raw_args = cfg.get("pytest_args", []) + if isinstance(raw_args, str): + pytest_args = shlex.split(raw_args) if raw_args else [] + elif isinstance(raw_args, list): + pytest_args = [str(a) for a in raw_args] + else: + raise TypeError( + f"'{workspace}.pytest_args' must be a list or string in " + f"{COLCON_UNIT_TEST_PACKAGES_YAML.name}, got {type(raw_args).__name__}" + ) + return packages, pytest_args def colcon_test_robot_command(workspace="robot"): @@ -83,9 +93,7 @@ def colcon_test_robot_command(workspace="robot"): if pytest_args: # One --pytest-args per token so flags like -p are not swallowed into # the -m expression (colcon forwards a single quoted blob as one argv). - cmd += "".join( - f" --pytest-args {shlex.quote(a)}" for a in shlex.split(pytest_args) - ) + cmd += "".join(f" --pytest-args {shlex.quote(a)}" for a in pytest_args) return cmd # Unit tests live co-located with their ROS 2 packages in robot/ros_ws/src/. # Thin proxy files under tests/robot/ re-export those tests so that diff --git a/tests/system/test_build_packages.py b/tests/system/test_build_packages.py index 5c54cfee3..5f3ca464c 100644 --- a/tests/system/test_build_packages.py +++ b/tests/system/test_build_packages.py @@ -1,3 +1,4 @@ +import shlex from pathlib import Path import pytest @@ -69,9 +70,11 @@ def test_colcon_test_robot(self): ) assert build.returncode == 0, f"colcon build (with testing) failed:\n{read_log_tail()}" + # shlex.quote the whole command so embedded --pytest-args quotes + # (e.g. 'not linter') are not eaten by the outer bash -ic quotes. test = docker_exec( container, - f"bash -ic '{colcon_test_robot_command('robot')}'", + f"bash -ic {shlex.quote(colcon_test_robot_command('robot'))}", timeout=300, ) assert test.returncode == 0, ( From 3d08099d299f6ecc3bcb24a6c84c2235ddcbcf79 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Wed, 12 Aug 2026 14:18:41 -0400 Subject: [PATCH 23/27] fix(ci): pass colcon pytest flags via PYTEST_ADDOPTS colcon --pytest-args is a single nargs='*' option, so repeating it dropped -p and pytest treated no:launch_testing as a file path. Set PYTEST_ADDOPTS with docker exec -e instead. Co-authored-by: Cursor --- tests/colcon_unit_test_packages.yaml | 2 +- tests/conftest.py | 31 +++++++++++++++++++--------- tests/system/test_build_packages.py | 11 ++++++---- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/tests/colcon_unit_test_packages.yaml b/tests/colcon_unit_test_packages.yaml index ff1a039fe..7fc67b97c 100644 --- a/tests/colcon_unit_test_packages.yaml +++ b/tests/colcon_unit_test_packages.yaml @@ -12,7 +12,7 @@ robot: # Skips ament_copyright / flake8 / pep257 on Python packages; run linters separately. # Disable launch_testing: these packages don't use it, and ROS Jazzy's plugin # still declares the removed pytest_pycollect_makemodule(path=...) hook arg. - # List form: each entry is one argv token forwarded via --pytest-args (see conftest). + # List form: each entry is one argv token, passed as PYTEST_ADDOPTS (see conftest). pytest_args: - -m - not linter diff --git a/tests/conftest.py b/tests/conftest.py index ed7a09ff3..ef0010c3e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,18 +83,24 @@ def load_colcon_unit_test_config(workspace="robot"): def colcon_test_robot_command(workspace="robot"): - """Shell command for colcon test over unit-test packages (robot workspace).""" - packages, pytest_args = load_colcon_unit_test_config(workspace) + """Shell command for colcon test over unit-test packages (robot workspace). + + Pytest flags from the YAML are *not* put on this command. colcon's + ``--pytest-args`` is a single nargs='*' option (last occurrence wins), + and nesting those tokens through ``bash -ic`` also breaks quoting. + Pass them as ``PYTEST_ADDOPTS`` via ``docker_exec(..., env=...)``. + """ + packages, _ = load_colcon_unit_test_config(workspace) pkg_list = " ".join(packages) - cmd = ( + return ( f"colcon test --packages-select {pkg_list} " "--event-handlers console_direct+ --return-code-on-test-failure" ) - if pytest_args: - # One --pytest-args per token so flags like -p are not swallowed into - # the -m expression (colcon forwards a single quoted blob as one argv). - cmd += "".join(f" --pytest-args {shlex.quote(a)}" for a in pytest_args) - return cmd + + +def pytest_addopts_env(pytest_args): + """Build a PYTEST_ADDOPTS value that pytest will shlex-split back to tokens.""" + return " ".join(shlex.quote(a) for a in pytest_args) # Unit tests live co-located with their ROS 2 packages in robot/ros_ws/src/. # Thin proxy files under tests/robot/ re-export those tests so that # `pytest tests/` and `airstack test -m unit` discover them without any @@ -355,8 +361,13 @@ def _run_teed(cmd_list, timeout, log_name=None, env=None, cwd=None): return result -def docker_exec(container, cmd, timeout=60, log_name=None): - full_cmd = ["docker", "exec", container, "bash", "-c", cmd] +def docker_exec(container, cmd, timeout=60, log_name=None, env=None): + """Run ``cmd`` in ``container``. ``env`` is passed as ``docker exec -e``.""" + full_cmd = ["docker", "exec"] + if env: + for key, value in env.items(): + full_cmd.extend(["-e", f"{key}={value}"]) + full_cmd.extend([container, "bash", "-c", cmd]) return _run_teed(full_cmd, timeout=timeout, log_name=log_name) diff --git a/tests/system/test_build_packages.py b/tests/system/test_build_packages.py index 5f3ca464c..acdd1784a 100644 --- a/tests/system/test_build_packages.py +++ b/tests/system/test_build_packages.py @@ -5,7 +5,7 @@ from conftest import (AIRSTACK_ROOT, airstack_cmd, colcon_test_robot_command, docker_exec, load_colcon_unit_test_config, logger, - read_log_tail, wait_for_container) + pytest_addopts_env, read_log_tail, wait_for_container) def _warn_if_prebuilt(*ws_paths): @@ -53,7 +53,7 @@ def test_colcon_test_robot(self): Package list and pytest args come from tests/colcon_unit_test_packages.yaml. Workspace-wide ament linter tests are not gated here. """ - packages, _ = load_colcon_unit_test_config("robot") + packages, pytest_args = load_colcon_unit_test_config("robot") try: result = airstack_cmd("up", "robot-desktop", env_overrides={"AUTOLAUNCH": "false", "DISPLAY": ""}, @@ -70,12 +70,15 @@ def test_colcon_test_robot(self): ) assert build.returncode == 0, f"colcon build (with testing) failed:\n{read_log_tail()}" - # shlex.quote the whole command so embedded --pytest-args quotes - # (e.g. 'not linter') are not eaten by the outer bash -ic quotes. + # PYTEST_ADDOPTS via docker exec -e: colcon --pytest-args cannot + # carry both -m and -p (last group wins), and bash -ic quoting + # eats tokens like 'not linter'. test = docker_exec( container, f"bash -ic {shlex.quote(colcon_test_robot_command('robot'))}", timeout=300, + env={"PYTEST_ADDOPTS": pytest_addopts_env(pytest_args)} + if pytest_args else None, ) assert test.returncode == 0, ( f"colcon test failed (packages: {', '.join(packages)}):\n{read_log_tail()}" From 51f2c8e32adf7028e6ad8af192a41f935d5811b2 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Wed, 12 Aug 2026 16:07:03 -0400 Subject: [PATCH 24/27] fix(ci): rename helper so pytest does not treat it as a hook conftest functions named pytest_* are registered as hooks. pytest_addopts_env caused PluginValidationError and exit code 3. Co-authored-by: Cursor --- tests/conftest.py | 8 ++++++-- tests/system/test_build_packages.py | 7 ++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ef0010c3e..01909bbf7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,8 +98,12 @@ def colcon_test_robot_command(workspace="robot"): ) -def pytest_addopts_env(pytest_args): - """Build a PYTEST_ADDOPTS value that pytest will shlex-split back to tokens.""" +def format_pytest_addopts(pytest_args): + """Build a PYTEST_ADDOPTS value that pytest will shlex-split back to tokens. + + Do not name this pytest_*: conftest functions with that prefix are treated + as pytest hooks and fail collection (exit code 3). + """ return " ".join(shlex.quote(a) for a in pytest_args) # Unit tests live co-located with their ROS 2 packages in robot/ros_ws/src/. # Thin proxy files under tests/robot/ re-export those tests so that diff --git a/tests/system/test_build_packages.py b/tests/system/test_build_packages.py index acdd1784a..0e95c8bf8 100644 --- a/tests/system/test_build_packages.py +++ b/tests/system/test_build_packages.py @@ -4,8 +4,9 @@ import pytest from conftest import (AIRSTACK_ROOT, airstack_cmd, colcon_test_robot_command, - docker_exec, load_colcon_unit_test_config, logger, - pytest_addopts_env, read_log_tail, wait_for_container) + docker_exec, format_pytest_addopts, + load_colcon_unit_test_config, logger, read_log_tail, + wait_for_container) def _warn_if_prebuilt(*ws_paths): @@ -77,7 +78,7 @@ def test_colcon_test_robot(self): container, f"bash -ic {shlex.quote(colcon_test_robot_command('robot'))}", timeout=300, - env={"PYTEST_ADDOPTS": pytest_addopts_env(pytest_args)} + env={"PYTEST_ADDOPTS": format_pytest_addopts(pytest_args)} if pytest_args else None, ) assert test.returncode == 0, ( From 9b8b28beaa7c0f0bec4c67c104c545cd35662857 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Wed, 12 Aug 2026 16:22:02 -0400 Subject: [PATCH 25/27] ci: skip image-build for build_packages reruns Pull and retag cache_* images instead of baking isaac/airsim on every colcon/pytest iteration. /pytest --no-image-build does the same for other marks. compose up --no-build when AIRSTACK_NO_IMAGE_BUILD=1. Co-authored-by: Cursor --- .agents/skills/run-system-tests/SKILL.md | 1 + .github/workflows/system-tests.yml | 81 ++++++++++++++++--- airstack.sh | 7 +- .../development/intermediate/testing/ci_cd.md | 7 ++ tests/conftest.py | 3 + 5 files changed, 89 insertions(+), 10 deletions(-) diff --git a/.agents/skills/run-system-tests/SKILL.md b/.agents/skills/run-system-tests/SKILL.md index f9b41b727..2605c5b7b 100644 --- a/.agents/skills/run-system-tests/SKILL.md +++ b/.agents/skills/run-system-tests/SKILL.md @@ -97,6 +97,7 @@ The `system-tests.yml` workflow's `Parse pytest args` step automatically prepend - `/pytest -m takeoff_hover_land` → effectively runs `-m "build_packages or takeoff_hover_land"` - `/pytest` (no marks) → pytest defaults (everything) - `/pytest -m build_docker` → unchanged (the build_docker tests rebuild from scratch anyway) +- `/pytest -m build_packages` → **pull-only** (retag `cache_*`, no `image-build`, no Isaac). Add `--no-image-build` on other marks to skip the bake. This guarantees that ROS 2 workspaces are built inside the containers before any launch/liveliness test tries to source them. If you intentionally want to skip `build_packages` (e.g. you trust the prebuilt images), include it explicitly: `-m "liveliness and not build_packages"` would work, but the simpler path is to run locally where the prepend logic doesn't apply. diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index f60240a44..1ce390ebe 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -152,16 +152,28 @@ jobs: print(f'::error::Could not parse pytest args from comment: {e}', file=sys.stderr) sys.exit(1) + # CI-only flag: do not forward to pytest. + no_image_build = False + stripped = [] + for a in args: + if a in ('--no-image-build', '--pull-only'): + no_image_build = True + else: + stripped.append(a) + args = stripped + # Pull out --sim and -m so the image-prep step can scope profiles # and decide whether to skip (build_docker tests rebuild themselves). # When --sim isn't given we mirror conftest's default so prep covers # whatever pytest will actually exercise. sim = 'msairsim,isaacsim' + sim_explicit = False marks = '' marks_idx = -1 for i, a in enumerate(args): if a == '--sim' and i + 1 < len(args): sim = args[i + 1] + sim_explicit = True elif a == '-m' and i + 1 < len(args): marks = args[i + 1] marks_idx = i + 1 @@ -174,6 +186,24 @@ jobs: marks = f'build_packages or {marks}' args[marks_idx] = marks + # colcon tests do not need Isaac. Default --sim would otherwise bake + # isaac-sim + ms-airsim (~1h) before a 1s colcon test. Pull registry + # cache tags instead; never image-build. + marks_norm = marks.replace('"', '').replace("'", '').strip() + args_blob = ' '.join(args) + heavy = any(m in marks_norm for m in ( + 'liveliness', 'sensors', 'takeoff_hover_land', 'autonomy', 'build_docker', + )) + only_packages = marks_norm == 'build_packages' or ( + not heavy and any(s in args_blob for s in ( + 'test_build_packages', 'test_colcon_', + )) + ) + if only_packages: + no_image_build = True + if not sim_explicit: + sim = 'msairsim' + skip_prep = 'build_docker' in marks quoted = ' '.join(shlex.quote(a) for a in args) @@ -181,10 +211,12 @@ jobs: f.write(f'pytest_args={quoted}\n') f.write(f'sim={sim}\n') f.write(f'skip_image_prep={"true" if skip_prep else "false"}\n') + f.write(f'no_image_build={"true" if no_image_build else "false"}\n') print(f'Resolved pytest args: {quoted or "(none — pytest defaults)"}') print(f'Resolved sim profile: {sim}') print(f'Skip image prep: {skip_prep}') + print(f'No image build (pull/retag only): {no_image_build}') PYEOF # Reply on the PR thread so the commenter sees their /pytest was @@ -200,7 +232,10 @@ jobs: const args = ${{ toJSON(steps.parse.outputs.pytest_args) }}; const cmd = `pytest tests/ ${args}`.trim(); const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const note = `Note: \`build_packages\` is automatically prepended whenever any marks are specified, to ensure code is built before launch tests run.`; + const pullOnly = '${{ steps.parse.outputs.no_image_build }}' === 'true'; + const note = pullOnly + ? `Note: pull-only image prep (no \`image-build\`). \`-m build_packages\` does not pull Isaac Sim. Add \`--no-image-build\` on other marks to skip rebuilds.` + : `Note: \`build_packages\` is automatically prepended whenever any marks are specified, to ensure code is built before launch tests run.`; await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, @@ -288,26 +323,30 @@ jobs: - name: Ensure airstack.sh is executable run: chmod +x airstack.sh + - name: Disable compose image builds + if: ${{ steps.parse.outputs.no_image_build == 'true' }} + run: echo "AIRSTACK_NO_IMAGE_BUILD=1" >> "$GITHUB_ENV" + # The ephemeral runner starts with no local images. `airstack_env` in # tests/conftest.py fails fast if compose images are missing, so prep # them here. Profile-gated services (ms-airsim, isaac-sim) are skipped # by compose unless their profile is active, so we mirror the fixture's - # profile selection from the parsed --sim. Pull-only by default; fall - # back to a full build only if the registry doesn't have everything - # (e.g. new branch with no published image yet). Skipped when the - # marks expression contains build_docker — those tests build per-service - # themselves. + # profile selection from the parsed --sim. Pull versioned tags, then + # retag floating cache_* tags onto the VERSION name (PR tags never + # exist). Fall back to image-build only when --no-image-build is off. + # Skipped when marks contain build_docker — those tests build themselves. - name: Ensure Docker images present if: ${{ steps.parse.outputs.skip_image_prep != 'true' }} env: AIRSTACK_ROOT: ${{ github.workspace }} SIM_INPUT: ${{ steps.parse.outputs.sim }} + NO_IMAGE_BUILD: ${{ steps.parse.outputs.no_image_build }} run: | profiles=desktop [[ ",$SIM_INPUT," == *,msairsim,* ]] && profiles="$profiles,ms-airsim" [[ ",$SIM_INPUT," == *,isaacsim,* ]] && profiles="$profiles,isaac-sim" export COMPOSE_PROFILES="$profiles" - echo "Pulling images for COMPOSE_PROFILES=$COMPOSE_PROFILES" + echo "Pulling images for COMPOSE_PROFILES=$COMPOSE_PROFILES (no_image_build=$NO_IMAGE_BUILD)" # Pull from registry; tolerate per-image failures so we can detect # what's still missing afterwards instead of aborting on the first @@ -315,6 +354,25 @@ jobs: # still surface on stderr. ./airstack.sh --progress=quiet image-pull --ignore-pull-failures || true + # VERSION tags miss on every PR. Seed from floating cache_* tags. + cache_tag="$(grep -E '^CACHE_TAG=' .env 2>/dev/null | cut -d= -f2 | tr -d '"' || true)" + cache_tag="${cache_tag:-cache}" + while IFS= read -r img; do + [[ -z "$img" ]] && continue + if docker image inspect "$img" --format '{{.Id}}' >/dev/null 2>&1; then + continue + fi + # Replace :v_ with :_ (PR versioned tags never exist) + cache_img="$(python3 -c "import re,sys; print(re.sub(r':v[^_]+_', f':{sys.argv[2]}_', sys.argv[1], count=1))" "$img" "$cache_tag")" + echo "Versioned tag missing; trying cache tag $cache_img" + if docker pull --quiet "$cache_img"; then + docker tag "$cache_img" "$img" + echo "Retagged $cache_img -> $img" + else + echo "Cache tag pull failed for $cache_img" + fi + done < <(docker compose -f docker-compose.yaml config --images) + missing=() while IFS= read -r img; do [[ -z "$img" ]] && continue @@ -324,11 +382,16 @@ jobs: done < <(docker compose -f docker-compose.yaml config --images) if (( ${#missing[@]} > 0 )); then - echo "Pull did not produce these images; falling back to build:" + echo "Images still missing after pull/retag:" printf ' - %s\n' "${missing[@]}" + if [[ "$NO_IMAGE_BUILD" == "true" ]]; then + echo "::error::Pull-only mode (--no-image-build or -m build_packages) will not image-build. Run /pytest -m build_docker once, or omit --no-image-build." + exit 1 + fi + echo "Falling back to image-build" ./airstack.sh --progress=quiet image-build else - echo "All required images present after pull — skipping build." + echo "All required images present after pull/retag — skipping build." fi - name: Run tests diff --git a/airstack.sh b/airstack.sh index a0a1d431c..d42c5b6e1 100755 --- a/airstack.sh +++ b/airstack.sh @@ -897,7 +897,12 @@ function cmd_up { fi log_info "Starting services..." - run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" up "${subcmd_args[@]}" -d + local up_opts=() + if [[ "${AIRSTACK_NO_IMAGE_BUILD:-}" == "1" ]]; then + log_info "AIRSTACK_NO_IMAGE_BUILD=1 → compose up --no-build" + up_opts+=(--no-build) + fi + run_docker_compose -f "$PROJECT_ROOT/docker-compose.yaml" "${global_args[@]}" up "${up_opts[@]}" "${subcmd_args[@]}" -d log_info "Services brought up successfully" } diff --git a/docs/development/intermediate/testing/ci_cd.md b/docs/development/intermediate/testing/ci_cd.md index 40a16af82..c20406739 100644 --- a/docs/development/intermediate/testing/ci_cd.md +++ b/docs/development/intermediate/testing/ci_cd.md @@ -198,6 +198,13 @@ The first line is parsed with `shlex`; everything after it is free-form notes. Checking whether the DDS bridge fix holds under 3 robots — see thread above. ``` +`-m build_packages` is **pull-only**: it retags floating `cache_*` images onto the PR `VERSION` tag and never runs `image-build` (and does not pull Isaac Sim). Use that when iterating on colcon/pytest failures. For other marks, add `--no-image-build` to skip the bake: + +```text +/pytest -m build_packages +/pytest -m liveliness --sim msairsim --no-image-build +``` + The workflow replies on the thread with the exact `pytest` command it resolved and a link to the run, and opens a **Check Run** pinned to the PR head SHA so comment-triggered runs still show up in the PR's Checks tab. diff --git a/tests/conftest.py b/tests/conftest.py index 01909bbf7..0edccf716 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -146,6 +146,9 @@ def pytest_addoption(parser): parser.addoption("--trajectory-types", default="Circle,Figure8,Racetrack,Line", help="Comma-separated fixed trajectory types to sweep in " "test_fixed_trajectory. Default: Circle,Figure8,Racetrack,Line") + parser.addoption("--no-image-build", action="store_true", default=False, + help="CI flag: skip image-build in system-tests.yml. " + "Ignored by pytest itself.") def pytest_configure(config): From 133555450c77baee876b1a567b123b1b6d354418 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Wed, 12 Aug 2026 18:55:08 -0400 Subject: [PATCH 26/27] fix(ci): disable pytest plugin autoload for colcon tests -p no:launch_testing is applied after setuptools entrypoints load, so pytest 8.1+ still crashes on launch_testing's path= hook. Set PYTEST_DISABLE_PLUGIN_AUTOLOAD so cache_* robot images (unpinned pytest) can run lidar tests without a rebuild. Co-authored-by: Cursor --- tests/colcon_unit_test_packages.yaml | 8 +++----- tests/system/test_build_packages.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/colcon_unit_test_packages.yaml b/tests/colcon_unit_test_packages.yaml index 7fc67b97c..767632c2a 100644 --- a/tests/colcon_unit_test_packages.yaml +++ b/tests/colcon_unit_test_packages.yaml @@ -10,11 +10,9 @@ robot: - natnet_ros2 - lidar_point_cloud_filter # Skips ament_copyright / flake8 / pep257 on Python packages; run linters separately. - # Disable launch_testing: these packages don't use it, and ROS Jazzy's plugin - # still declares the removed pytest_pycollect_makemodule(path=...) hook arg. - # List form: each entry is one argv token, passed as PYTEST_ADDOPTS (see conftest). + # launch_testing is disabled via PYTEST_DISABLE_PLUGIN_AUTOLOAD in the test + # ( -p no:launch_testing is too late: pytest 8.1+ validates the plugin at + # register, before -p is applied). pytest_args: - -m - not linter - - -p - - no:launch_testing diff --git a/tests/system/test_build_packages.py b/tests/system/test_build_packages.py index 0e95c8bf8..6f5d0715f 100644 --- a/tests/system/test_build_packages.py +++ b/tests/system/test_build_packages.py @@ -71,15 +71,18 @@ def test_colcon_test_robot(self): ) assert build.returncode == 0, f"colcon build (with testing) failed:\n{read_log_tail()}" - # PYTEST_ADDOPTS via docker exec -e: colcon --pytest-args cannot - # carry both -m and -p (last group wins), and bash -ic quoting - # eats tokens like 'not linter'. + # PYTEST_ADDOPTS via docker exec -e (not colcon --pytest-args). + # PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 skips launch_testing before + # pytest 8.1+ validates its removed path= hook. -p no:launch_testing + # is too late. Needed on cache_* images that predate the pytest<8.1 pin. + exec_env = {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"} + if pytest_args: + exec_env["PYTEST_ADDOPTS"] = format_pytest_addopts(pytest_args) test = docker_exec( container, f"bash -ic {shlex.quote(colcon_test_robot_command('robot'))}", timeout=300, - env={"PYTEST_ADDOPTS": format_pytest_addopts(pytest_args)} - if pytest_args else None, + env=exec_env, ) assert test.returncode == 0, ( f"colcon test failed (packages: {', '.join(packages)}):\n{read_log_tail()}" From 6a28d20cf7c687a30edb00a571ee057889a6b5b9 Mon Sep 17 00:00:00 2001 From: Pranav Kumara Date: Wed, 12 Aug 2026 22:48:59 -0400 Subject: [PATCH 27/27] fix(ci): skip lidar ament linters in package pytest config PYTEST_ADDOPTS -m not linter never reached ament pytest, so copyright / flake8 / pep257 still ran after the unit tests passed. Ignore those modules in setup.cfg and collect_ignore. Co-authored-by: Cursor --- .../src/sensors/lidar_point_cloud_filter/setup.cfg | 10 ++++++++++ .../sensors/lidar_point_cloud_filter/test/conftest.py | 7 +++++++ tests/colcon_unit_test_packages.yaml | 11 ++++------- tests/system/test_build_packages.py | 6 +++--- 4 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/conftest.py diff --git a/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg index 55f87f13c..7e02f4b35 100644 --- a/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg +++ b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/setup.cfg @@ -10,3 +10,13 @@ python_classes = Test* python_functions = test_* markers = unit: Hermetic unit tests (no ROS stack required) + linter: ament copyright/flake8/pep257 (run separately, not via colcon test) + copyright: ament_copyright + flake8: ament_flake8 + pep257: ament_pep257 +# colcon test / PYTEST_ADDOPTS -m is dropped by ament pytest. Ignore linter +# modules here so only unit tests run. +addopts = + --ignore=test/test_copyright.py + --ignore=test/test_flake8.py + --ignore=test/test_pep257.py diff --git a/robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/conftest.py b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/conftest.py new file mode 100644 index 000000000..c985d5923 --- /dev/null +++ b/robot/ros_ws/src/sensors/lidar_point_cloud_filter/test/conftest.py @@ -0,0 +1,7 @@ +# Skip ament linter modules during pytest/colcon test. +# PYTEST_ADDOPTS -m is not forwarded by ament pytest; collect_ignore is. +collect_ignore = [ + "test_copyright.py", + "test_flake8.py", + "test_pep257.py", +] diff --git a/tests/colcon_unit_test_packages.yaml b/tests/colcon_unit_test_packages.yaml index 767632c2a..07e06a5b4 100644 --- a/tests/colcon_unit_test_packages.yaml +++ b/tests/colcon_unit_test_packages.yaml @@ -9,10 +9,7 @@ robot: packages: - natnet_ros2 - lidar_point_cloud_filter - # Skips ament_copyright / flake8 / pep257 on Python packages; run linters separately. - # launch_testing is disabled via PYTEST_DISABLE_PLUGIN_AUTOLOAD in the test - # ( -p no:launch_testing is too late: pytest 8.1+ validates the plugin at - # register, before -p is applied). - pytest_args: - - -m - - not linter + # Linter skip lives in lidar_point_cloud_filter setup.cfg + test/conftest.py. + # ament pytest does not honor PYTEST_ADDOPTS -m. + # launch_testing is skipped via PYTEST_DISABLE_PLUGIN_AUTOLOAD in the test. + pytest_args: [] diff --git a/tests/system/test_build_packages.py b/tests/system/test_build_packages.py index 6f5d0715f..aa84b0464 100644 --- a/tests/system/test_build_packages.py +++ b/tests/system/test_build_packages.py @@ -71,10 +71,10 @@ def test_colcon_test_robot(self): ) assert build.returncode == 0, f"colcon build (with testing) failed:\n{read_log_tail()}" - # PYTEST_ADDOPTS via docker exec -e (not colcon --pytest-args). # PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 skips launch_testing before - # pytest 8.1+ validates its removed path= hook. -p no:launch_testing - # is too late. Needed on cache_* images that predate the pytest<8.1 pin. + # pytest 8.1+ validates its removed path= hook. Linter skip is in + # the package setup.cfg / test/conftest.py (ament pytest ignores + # PYTEST_ADDOPTS -m). exec_env = {"PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"} if pytest_args: exec_env["PYTEST_ADDOPTS"] = format_pytest_addopts(pytest_args)