From da6449dc3285bf4904e375e3d4070c7cfdbd3ff3 Mon Sep 17 00:00:00 2001 From: Blue <3067670134@qq.com> Date: Sat, 3 Oct 2026 21:36:50 +0800 Subject: [PATCH 1/2] fix(examples): stop launchers when gateway readiness fails --- examples/calc_x/run_local.sh | 10 +- examples/calc_x/run_minikube.sh | 10 +- examples/gsm8k/run_local.sh | 10 +- examples/llm-in-sandbox/run.sh | 8 +- examples/science_world/run_local.sh | 10 +- examples/search_r1/run.sh | 10 +- tests/examples/test_launcher_readiness.py | 153 ++++++++++++++++++++++ 7 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 tests/examples/test_launcher_readiness.py diff --git a/examples/calc_x/run_local.sh b/examples/calc_x/run_local.sh index bb753f914..2d95353d2 100755 --- a/examples/calc_x/run_local.sh +++ b/examples/calc_x/run_local.sh @@ -34,10 +34,18 @@ agl-server \ default_proxy.model_name=Qwen/Qwen2.5-1.5B-Instruct \ >"$SERVER_LOG" 2>&1 & +server_ready=0 for _ in $(seq 1 60); do - curl -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null && break + if curl --max-time 1 -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null; then + server_ready=1 + break + fi sleep 1 done +if [[ "$server_ready" -ne 1 ]]; then + echo "Agent Lightning server did not become ready after 60 attempts; see $SERVER_LOG" >&2 + exit 1 +fi agl-controller \ runner_type=local \ diff --git a/examples/calc_x/run_minikube.sh b/examples/calc_x/run_minikube.sh index effcc024e..2a3286058 100755 --- a/examples/calc_x/run_minikube.sh +++ b/examples/calc_x/run_minikube.sh @@ -36,10 +36,18 @@ agl-server \ default_proxy.model_name=Qwen/Qwen2.5-1.5B-Instruct \ >"$SERVER_LOG" 2>&1 & +server_ready=0 for _ in $(seq 1 60); do - curl -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null && break + if curl --max-time 1 -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null; then + server_ready=1 + break + fi sleep 1 done +if [[ "$server_ready" -ne 1 ]]; then + echo "Agent Lightning server did not become ready after 60 attempts; see $SERVER_LOG" >&2 + exit 1 +fi agl-controller \ runner_type=k8s \ diff --git a/examples/gsm8k/run_local.sh b/examples/gsm8k/run_local.sh index f0e0ac3a1..0efcb0e3b 100755 --- a/examples/gsm8k/run_local.sh +++ b/examples/gsm8k/run_local.sh @@ -33,10 +33,18 @@ agl-server \ default_proxy.model_name=Qwen/Qwen2.5-1.5B-Instruct \ >"$SERVER_LOG" 2>&1 & +server_ready=0 for _ in $(seq 1 60); do - curl -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null && break + if curl --max-time 1 -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null; then + server_ready=1 + break + fi sleep 1 done +if [[ "$server_ready" -ne 1 ]]; then + echo "Agent Lightning server did not become ready after 60 attempts; see $SERVER_LOG" >&2 + exit 1 +fi agl-controller \ runner_type=local \ diff --git a/examples/llm-in-sandbox/run.sh b/examples/llm-in-sandbox/run.sh index 9a7d44ee5..8ee308c77 100755 --- a/examples/llm-in-sandbox/run.sh +++ b/examples/llm-in-sandbox/run.sh @@ -33,12 +33,18 @@ agl-server \ default_proxy.model_name=Qwen/Qwen3-4B-Instruct-2507 & echo "=== Waiting for Agent Lightning server ===" +server_ready=0 for _ in $(seq 1 60); do - if curl -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null 2>&1; then + if curl --max-time 1 -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null 2>&1; then + server_ready=1 break fi sleep 1 done +if [[ "$server_ready" -ne 1 ]]; then + echo "Agent Lightning server did not become ready on port $AGL_SERVER_PORT after 60 attempts" >&2 + exit 1 +fi echo "=== Starting Agent Lightning controller ===" agl-controller \ diff --git a/examples/science_world/run_local.sh b/examples/science_world/run_local.sh index 2d5263f2b..ac9889e07 100755 --- a/examples/science_world/run_local.sh +++ b/examples/science_world/run_local.sh @@ -30,10 +30,18 @@ agl-server \ key="$AGL_KEY" \ default_proxy.model_name=Qwen/Qwen2.5-7B-Instruct & +server_ready=0 for _ in $(seq 1 60); do - curl -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null && break + if curl --max-time 1 -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null; then + server_ready=1 + break + fi sleep 1 done +if [[ "$server_ready" -ne 1 ]]; then + echo "Agent Lightning server did not become ready on port $AGL_SERVER_PORT after 60 attempts" >&2 + exit 1 +fi agl-controller \ runner_type=local \ diff --git a/examples/search_r1/run.sh b/examples/search_r1/run.sh index 84965cc53..a8b63d655 100755 --- a/examples/search_r1/run.sh +++ b/examples/search_r1/run.sh @@ -24,10 +24,18 @@ agl-server \ key="$AGL_KEY" \ default_proxy.model_name="$SEARCH_R1_MODEL" & +server_ready=0 for _ in $(seq 1 60); do - curl -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null && break + if curl --max-time 1 -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null; then + server_ready=1 + break + fi sleep 1 done +if [[ "$server_ready" -ne 1 ]]; then + echo "Agent Lightning server did not become ready on port $AGL_SERVER_PORT after 60 attempts" >&2 + exit 1 +fi agl-controller \ runner_type=local \ diff --git a/tests/examples/test_launcher_readiness.py b/tests/examples/test_launcher_readiness.py new file mode 100644 index 000000000..c424f142a --- /dev/null +++ b/tests/examples/test_launcher_readiness.py @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""CPU-only readiness regression coverage for the example launchers.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parents[2] +BASH = shutil.which("bash") +LINUX_ONLY = pytest.mark.skipif(os.name == "nt" or BASH is None, reason="requires Linux bash") + + +@dataclass(frozen=True) +class Launcher: + path: str + cwd: str + + +LAUNCHERS = [ + Launcher("examples/calc_x/run_local.sh", "examples/calc_x"), + Launcher("examples/calc_x/run_minikube.sh", "examples/calc_x"), + Launcher("examples/gsm8k/run_local.sh", "examples/gsm8k"), + Launcher("examples/science_world/run_local.sh", "."), + Launcher("examples/search_r1/run.sh", "."), + Launcher("examples/llm-in-sandbox/run.sh", "."), +] + + +def _write_executable(path: Path, content: str) -> None: + path.write_text(content, encoding="utf-8", newline="\n") + path.chmod(0o755) + + +def _copy_launcher(tmp_path: Path, launcher: Launcher) -> tuple[Path, Path]: + root = tmp_path / "repo" + target = root / launcher.path + target.parent.mkdir(parents=True) + target.write_text((REPO_ROOT / launcher.path).read_text(encoding="utf-8"), encoding="utf-8", newline="\n") + target.chmod(0o755) + cwd = root if launcher.cwd == "." else root / launcher.cwd + cwd.mkdir(parents=True, exist_ok=True) + return target, cwd + + +def _mock_path(tmp_path: Path) -> Path: + tools = tmp_path / "bin" + tools.mkdir() + for name in ("date", "dirname", "env", "seq"): + executable = shutil.which(name) + assert executable is not None + (tools / name).symlink_to(executable) + + command = """#!/bin/sh +line="${0##*/}" +for argument in "$@"; do + line="$line <$argument>" +done +printf '%s\n' "$line" >> "$MOCK_COMMAND_LOG" +exit 0 +""" + for name in ("agl-controller", "agl-server", "minikube", "pkill", "python", "ray", "sleep"): + _write_executable(tools / name, command) + + _write_executable( + tools / "curl", + """#!/bin/sh +count=0 +if [ -f "$MOCK_CURL_COUNT" ]; then + read -r count < "$MOCK_CURL_COUNT" +fi +count=$((count + 1)) +printf '%s\n' "$count" > "$MOCK_CURL_COUNT" +printf '%s\n' "$*" >> "$MOCK_CURL_LOG" +if [ "$MOCK_CURL_SUCCEED_AT" -gt 0 ] && [ "$count" -ge "$MOCK_CURL_SUCCEED_AT" ]; then + exit 0 +fi +exit 22 +""", + ) + return tools + + +def _run( + tmp_path: Path, launcher: Launcher, *, succeed_at: int +) -> tuple[subprocess.CompletedProcess[str], list[str], list[str]]: + assert BASH is not None + script, cwd = _copy_launcher(tmp_path, launcher) + command_log = tmp_path / "commands.log" + curl_log = tmp_path / "curl.log" + env = { + **os.environ, + "PATH": str(_mock_path(tmp_path)), + "MOCK_COMMAND_LOG": str(command_log), + "MOCK_CURL_COUNT": str(tmp_path / "curl-count"), + "MOCK_CURL_LOG": str(curl_log), + "MOCK_CURL_SUCCEED_AT": str(succeed_at), + } + result = subprocess.run( + [BASH, str(script), "--marker", "value"], + cwd=cwd, + env=env, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + commands = command_log.read_text(encoding="utf-8").splitlines() + curls = curl_log.read_text(encoding="utf-8").splitlines() + return result, commands, curls + + +@LINUX_ONLY +@pytest.mark.parametrize("launcher", LAUNCHERS, ids=lambda launcher: launcher.path) +def test_launcher_stops_when_server_never_becomes_ready(tmp_path: Path, launcher: Launcher) -> None: + result, commands, curls = _run(tmp_path, launcher, succeed_at=0) + + assert result.returncode != 0 + assert "Agent Lightning server did not become ready" in result.stderr + assert not any(command.startswith("agl-controller") for command in commands) + assert not any(command.startswith("python ") for command in commands) + assert (tmp_path / "curl-count").read_text(encoding="utf-8").strip() == "60" + assert all("--max-time 1" in call for call in curls) + + +@LINUX_ONLY +@pytest.mark.parametrize("launcher", LAUNCHERS, ids=lambda launcher: launcher.path) +def test_launcher_continues_after_later_success_and_preserves_arguments(tmp_path: Path, launcher: Launcher) -> None: + result, commands, curls = _run(tmp_path, launcher, succeed_at=3) + + assert result.returncode == 0, result.stderr + assert any(command.startswith("agl-controller") for command in commands) + trainer = next(command for command in commands if command.startswith("python ")) + assert trainer.endswith(" <--marker> ") + assert (tmp_path / "curl-count").read_text(encoding="utf-8").strip() == "3" + assert all("--max-time 1" in call for call in curls) + + +@LINUX_ONLY +def test_launcher_continues_after_first_success(tmp_path: Path) -> None: + result, commands, curls = _run(tmp_path, LAUNCHERS[0], succeed_at=1) + + assert result.returncode == 0, result.stderr + assert any(command.startswith("agl-controller") for command in commands) + assert any(command.startswith("python ") for command in commands) + assert (tmp_path / "curl-count").read_text(encoding="utf-8").strip() == "1" + assert all("--max-time 1" in call for call in curls) From 52b46c3047d0151be2629e83c3865dee84ff9cd9 Mon Sep 17 00:00:00 2001 From: Blue <3067670134@qq.com> Date: Sat, 3 Oct 2026 22:45:43 +0800 Subject: [PATCH 2/2] fix(examples): gate multimodal launcher on gateway readiness --- examples/multimodal_qa/run_local.sh | 10 +++++++++- tests/examples/test_launcher_readiness.py | 5 +++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/multimodal_qa/run_local.sh b/examples/multimodal_qa/run_local.sh index a0a8223d9..9c5bad50b 100755 --- a/examples/multimodal_qa/run_local.sh +++ b/examples/multimodal_qa/run_local.sh @@ -34,10 +34,18 @@ agl-server \ default_proxy.model_name="$MODEL" \ >"$SERVER_LOG" 2>&1 & +server_ready=0 for _ in $(seq 1 60); do - curl -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null && break + if curl --max-time 1 -sf "http://localhost:$AGL_SERVER_PORT/healthz" >/dev/null; then + server_ready=1 + break + fi sleep 1 done +if [[ "$server_ready" -ne 1 ]]; then + echo "Agent Lightning server did not become ready after 60 attempts; see $SERVER_LOG" >&2 + exit 1 +fi agl-controller \ runner_type=local \ diff --git a/tests/examples/test_launcher_readiness.py b/tests/examples/test_launcher_readiness.py index c424f142a..a8d6d0582 100644 --- a/tests/examples/test_launcher_readiness.py +++ b/tests/examples/test_launcher_readiness.py @@ -27,6 +27,7 @@ class Launcher: Launcher("examples/calc_x/run_local.sh", "examples/calc_x"), Launcher("examples/calc_x/run_minikube.sh", "examples/calc_x"), Launcher("examples/gsm8k/run_local.sh", "examples/gsm8k"), + Launcher("examples/multimodal_qa/run_local.sh", "examples/multimodal_qa"), Launcher("examples/science_world/run_local.sh", "."), Launcher("examples/search_r1/run.sh", "."), Launcher("examples/llm-in-sandbox/run.sh", "."), @@ -103,7 +104,7 @@ def _run( "MOCK_CURL_SUCCEED_AT": str(succeed_at), } result = subprocess.run( - [BASH, str(script), "--marker", "value"], + [BASH, str(script), "--marker", "value with spaces"], cwd=cwd, env=env, capture_output=True, @@ -137,7 +138,7 @@ def test_launcher_continues_after_later_success_and_preserves_arguments(tmp_path assert result.returncode == 0, result.stderr assert any(command.startswith("agl-controller") for command in commands) trainer = next(command for command in commands if command.startswith("python ")) - assert trainer.endswith(" <--marker> ") + assert trainer.endswith(" <--marker> ") assert (tmp_path / "curl-count").read_text(encoding="utf-8").strip() == "3" assert all("--max-time 1" in call for call in curls)