From 840dbe11e5a0c6af298dd48b935cb4f50823dda9 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Tue, 27 Oct 2026 19:05:22 +0100 Subject: [PATCH 1/6] Migrate documentation to Fern Signed-off-by: Ivan Podkidyshev --- .github/workflows/ci.yml | 8 - .github/workflows/docs.yml | 106 +-- .gitignore | 3 +- CONTRIBUTING.md | 20 + doc/DEV.mdx | 99 +++ doc/DEV.rst | 98 --- doc/Makefile | 65 +- doc/Tutorial.mdx | 272 ++++++++ doc/Tutorial.rst | 283 -------- doc/USER_GUIDE.mdx | 580 +++++++++++++++++ doc/USER_GUIDE.rst | 613 ------------------ doc/api.mdx | 28 + doc/api.rst | 49 -- doc/conf.py | 114 ---- doc/generate_reference.py | 163 +++++ doc/index.mdx | 210 ++++++ doc/index.rst | 227 ------- doc/pareto-frontiers.mdx | 111 ++++ doc/pareto-frontiers.rst | 149 ----- doc/reporting.mdx | 369 +++++++++++ doc/reporting.rst | 322 --------- doc/{reports.rst => reports.mdx} | 8 +- doc/systems.mdx | 53 ++ doc/systems.rst | 67 -- doc/templates/reference.mdx.jinja | 56 ++ doc/training-report-schema.mdx | 60 ++ doc/training-report-schema.rst | 67 -- doc/troubleshooting.mdx | 28 + doc/troubleshooting.rst | 33 - doc/workloads/ai_dynamo.mdx | 443 +++++++++++++ doc/workloads/ai_dynamo.rst | 517 --------------- doc/workloads/aiconfigurator.mdx | 110 ++++ doc/workloads/aiconfigurator.rst | 112 ---- doc/workloads/bash_cmd.mdx | 67 ++ doc/workloads/bash_cmd.rst | 68 -- doc/workloads/chakra_replay.mdx | 65 ++ doc/workloads/chakra_replay.rst | 66 -- doc/workloads/ddlb.mdx | 75 +++ doc/workloads/ddlb.rst | 76 --- doc/workloads/deepep.mdx | 107 +++ doc/workloads/deepep.rst | 107 --- doc/workloads/dynamo_mocker.mdx | 117 ++++ doc/workloads/dynamo_mocker.rst | 125 ---- doc/workloads/fio.mdx | 120 ++++ doc/workloads/fio.rst | 131 ---- doc/workloads/index.mdx | 47 ++ doc/workloads/index.rst | 53 -- doc/workloads/jax_toolbox.mdx | 28 + doc/workloads/jax_toolbox.rst | 28 - doc/workloads/megatron_bridge.mdx | 212 ++++++ doc/workloads/megatron_bridge.rst | 264 -------- doc/workloads/nccl.mdx | 67 ++ doc/workloads/nccl.rst | 68 -- doc/workloads/nemo_launcher.mdx | 84 +++ doc/workloads/nemo_launcher.rst | 83 --- doc/workloads/nemo_run.mdx | 67 ++ doc/workloads/nemo_run.rst | 68 -- doc/workloads/nixl_bench.mdx | 138 ++++ doc/workloads/nixl_bench.rst | 159 ----- doc/workloads/nixl_ep.mdx | 109 ++++ doc/workloads/nixl_ep.rst | 106 --- doc/workloads/nixl_kvbench.mdx | 76 +++ doc/workloads/nixl_kvbench.rst | 76 --- doc/workloads/nixl_perftest.mdx | 86 +++ doc/workloads/nixl_perftest.rst | 86 --- doc/workloads/osu.mdx | 78 +++ doc/workloads/osu.rst | 84 --- doc/workloads/sglang.mdx | 267 ++++++++ doc/workloads/sglang.rst | 260 -------- doc/workloads/sleep.mdx | 65 ++ doc/workloads/sleep.rst | 66 -- doc/workloads/slurm_container.mdx | 75 +++ doc/workloads/slurm_container.rst | 80 --- doc/workloads/ucc.mdx | 65 ++ doc/workloads/ucc.rst | 66 -- doc/workloads/vllm.mdx | 285 ++++++++ doc/workloads/vllm.rst | 282 -------- ...> workloads_requirements_installation.mdx} | 41 +- fern/docs.yml | 125 ++++ fern/fern.config.json | 4 + fern/package-lock.json | 210 ++++++ fern/package.json | 11 + fern/reference.yml | 214 ++++++ pyproject.toml | 16 +- src/cloudai/models/__init__.py | 15 + src/cloudai/models/output.py | 2 +- src/cloudai/workloads/common/__init__.py | 15 + tests/doc/test_generate_reference.py | 97 +++ uv.lock | 446 +------------ 89 files changed, 5696 insertions(+), 5645 deletions(-) create mode 100644 doc/DEV.mdx delete mode 100644 doc/DEV.rst create mode 100644 doc/Tutorial.mdx delete mode 100644 doc/Tutorial.rst create mode 100644 doc/USER_GUIDE.mdx delete mode 100644 doc/USER_GUIDE.rst create mode 100644 doc/api.mdx delete mode 100644 doc/api.rst delete mode 100644 doc/conf.py create mode 100644 doc/generate_reference.py create mode 100644 doc/index.mdx delete mode 100644 doc/index.rst create mode 100644 doc/pareto-frontiers.mdx delete mode 100644 doc/pareto-frontiers.rst create mode 100644 doc/reporting.mdx delete mode 100644 doc/reporting.rst rename doc/{reports.rst => reports.mdx} (53%) create mode 100644 doc/systems.mdx delete mode 100644 doc/systems.rst create mode 100644 doc/templates/reference.mdx.jinja create mode 100644 doc/training-report-schema.mdx delete mode 100644 doc/training-report-schema.rst create mode 100644 doc/troubleshooting.mdx delete mode 100644 doc/troubleshooting.rst create mode 100644 doc/workloads/ai_dynamo.mdx delete mode 100644 doc/workloads/ai_dynamo.rst create mode 100644 doc/workloads/aiconfigurator.mdx delete mode 100644 doc/workloads/aiconfigurator.rst create mode 100644 doc/workloads/bash_cmd.mdx delete mode 100644 doc/workloads/bash_cmd.rst create mode 100644 doc/workloads/chakra_replay.mdx delete mode 100644 doc/workloads/chakra_replay.rst create mode 100644 doc/workloads/ddlb.mdx delete mode 100644 doc/workloads/ddlb.rst create mode 100644 doc/workloads/deepep.mdx delete mode 100644 doc/workloads/deepep.rst create mode 100644 doc/workloads/dynamo_mocker.mdx delete mode 100644 doc/workloads/dynamo_mocker.rst create mode 100644 doc/workloads/fio.mdx delete mode 100644 doc/workloads/fio.rst create mode 100644 doc/workloads/index.mdx delete mode 100644 doc/workloads/index.rst create mode 100644 doc/workloads/jax_toolbox.mdx delete mode 100644 doc/workloads/jax_toolbox.rst create mode 100644 doc/workloads/megatron_bridge.mdx delete mode 100644 doc/workloads/megatron_bridge.rst create mode 100644 doc/workloads/nccl.mdx delete mode 100644 doc/workloads/nccl.rst create mode 100644 doc/workloads/nemo_launcher.mdx delete mode 100644 doc/workloads/nemo_launcher.rst create mode 100644 doc/workloads/nemo_run.mdx delete mode 100644 doc/workloads/nemo_run.rst create mode 100644 doc/workloads/nixl_bench.mdx delete mode 100644 doc/workloads/nixl_bench.rst create mode 100644 doc/workloads/nixl_ep.mdx delete mode 100644 doc/workloads/nixl_ep.rst create mode 100644 doc/workloads/nixl_kvbench.mdx delete mode 100644 doc/workloads/nixl_kvbench.rst create mode 100644 doc/workloads/nixl_perftest.mdx delete mode 100644 doc/workloads/nixl_perftest.rst create mode 100644 doc/workloads/osu.mdx delete mode 100644 doc/workloads/osu.rst create mode 100644 doc/workloads/sglang.mdx delete mode 100644 doc/workloads/sglang.rst create mode 100644 doc/workloads/sleep.mdx delete mode 100644 doc/workloads/sleep.rst create mode 100644 doc/workloads/slurm_container.mdx delete mode 100644 doc/workloads/slurm_container.rst create mode 100644 doc/workloads/ucc.mdx delete mode 100644 doc/workloads/ucc.rst create mode 100644 doc/workloads/vllm.mdx delete mode 100644 doc/workloads/vllm.rst rename doc/workloads/{workloads_requirements_installation.rst => workloads_requirements_installation.mdx} (61%) create mode 100644 fern/docs.yml create mode 100644 fern/fern.config.json create mode 100644 fern/package-lock.json create mode 100644 fern/package.json create mode 100644 fern/reference.yml create mode 100644 src/cloudai/models/__init__.py create mode 100644 src/cloudai/workloads/common/__init__.py create mode 100644 tests/doc/test_generate_reference.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f452349c1..449379f5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,14 +30,6 @@ jobs: - name: Run linting via pre-commit run: uv run --locked pre-commit run --all-files --show-diff-on-failure - - name: Build documentation - run: | - set -eE - set -o pipefail - - uv run --locked --extra docs sphinx-build -M html "doc" "doc/_build" - touch doc/_build/html/.nojekyll - test: name: Run pytest (3.14) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 083e9d8dc..b7dac2525 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,81 +1,41 @@ -name: Build and Deploy Documentation +name: Validate Fern Documentation on: + pull_request: push: - branches: [ main ] - tags: [ 'v*' ] - workflow_dispatch: # Allow manual triggering + branches: [main] + tags: ['v*'] + workflow_dispatch: -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false +permissions: + contents: read jobs: - build: + docs: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - name: Build documentation - run: make -C doc/ html - - - name: Upload artifact - uses: actions/upload-pages-artifact@v4 - with: - path: ./doc/_build/html - - build-cms: - name: Build CMS Documentation - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: '24' - - - name: Build CMS documentation - run: make -C doc/ cms-html - - - name: Sanitize artifact name - id: artifact-name - run: echo "name=cloudai-docs-cms-$(echo '${{ github.ref_name }}' | tr '/' '-')" >> $GITHUB_OUTPUT - - - name: Upload CMS docs artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: doc/_build-cms/html - - deploy: - name: Deploy to GitHub Pages - needs: build - - permissions: - pages: write - id-token: write - - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - runs-on: ubuntu-latest - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version-file: .python-version + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - uses: actions/setup-node@v6 + with: + node-version: '24' + cache: npm + cache-dependency-path: fern/package-lock.json + - name: Generate and validate documentation + run: make -C doc check + - name: Test reference generation + run: uv run --locked --extra dev --extra docs pytest tests/doc/test_generate_reference.py + - name: Upload Fern sources + uses: actions/upload-artifact@v4 + with: + name: cloudai-fern-docs + path: | + doc/**/*.mdx + fern/docs.yml + fern/fern.config.json + fern/generated/ diff --git a/.gitignore b/.gitignore index 2113115d0..aa2f5d7d1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,8 @@ wheels/ share/python-wheels/ *.egg-info/ *.egg -doc/_build +fern/generated/ +fern/node_modules/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07e109027..510993ba7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,26 @@ # CloudAI OSS Contribution Guidelines Welcome to the CloudAI project! This document outlines the procedures and standards for contributing to the CloudAI source code. Adherence to these guidelines ensures code quality, readability, and maintainability. We look forward to your contributions! +## Documentation + +Documentation is authored in `doc/*.mdx` and `doc/workloads/*.mdx` and built with Fern. +Keep the README separate. Navigation lives in `fern/docs.yml`. + +With Python 3.14, Node.js 24 and uv installed, run `make -C doc check` to generate the Python reference +and validate the documentation, or `make -C doc preview` to start a local preview. +After changing Python docstrings or model fields, rerun `make -C doc reference`. +The preview uses Fern's legacy renderer because the default Astro preview currently returns +404s for compiled MDX modules, causing page content to disappear after loading. + +Generated reference snippets live in `fern/generated/` and are not checked in. Add classes +and member exclusions to `fern/reference.yml`; include their snippets in authored pages with +``. The build reads Python objects +and renders `doc/templates/reference.mdx.jinja`; it does not transform authored MDX. + +CI validates the docs and produces a Fern source artifact. Publishing requires the provisioned +Fern instance URL (`FERN_DOCS_URL`) and the enterprise deployment environment. The default URL +is for local validation only; this repository does not publish automatically. + ## Contribution Process ### Identifying the Feature or Fix Contributors looking to add features or fixes are encouraged to reach out to the team by creating issues on the project's issue tracker. This step ensures that efforts are not duplicated and that the proposed changes align with the project's goals. Please communicate with the main developers before starting work to avoid the potential for your PR not being accepted. diff --git a/doc/DEV.mdx b/doc/DEV.mdx new file mode 100644 index 000000000..dd4d03a6f --- /dev/null +++ b/doc/DEV.mdx @@ -0,0 +1,99 @@ + + +This chapter targets developers who want to contribute to the project's core. + +``` mermaid +graph TD + subgraph _core + base_modules + core_implementations + registry + end + + subgraph runners + SlurmRunner + StandaloneRunner + end + + subgraph installers + SlurmInstaller + StandaloneInstaller + end + + subgraph systems + SlurmSystem + StandaloneSystem + end + + installers --> _core + runners --> _core + systems --> _core +``` + + + +## Core Modules + +We use [import-linter](https://github.com/seddonym/import-linter) to ensure no core modules import higher level modules. + +`Registry` object is a singleton that holds implementation mappings. Users can register their own implementations to the registry or replace the default implementations. + + + +## Cache + +Some prerequisites can be installed. For example: + +Docker images, git repos with executable scripts, etc. All such "installables" are kept under the system's `install_path`. + +Installables are shared among all tests. Therefore, if any number of tests use the same installable, it is installed only once for a particular system TOML. + +`PythonExecutable` installs a Python project into its own virtual environment. When the selected +`pyproject.toml` has an adjacent `uv.lock`, it uses bundled uv to install the locked base dependencies +and the project, without extras or dependency groups. An outdated lockfile fails installation instead +of being updated. Projects without a lockfile continue to use pip. When `dependencies_from_pyproject=False` +selects an existing `requirements.txt`, that selection takes precedence over the lockfile. + +``` mermaid +classDiagram + class Installable { + <> + + __eq__(other: object) + + __hash__() + } + + class DockerImage { + + url: str + + install_path: str | Path + } + + class GitRepo { + + git_url: str + + commit_hash: str + + install_path: Path + } + + class PythonExecutable { + + git_repo: GitRepo + + venv_path: Path + } + + Installable <|-- DockerImage + Installable <|-- GitRepo + Installable <|-- PythonExecutable + PythonExecutable --> GitRepo + + class BaseInstaller { + <> + + install(items: Iterable[Installable]) + + uninstall(items: Iterable[Installable]) + + is_installed(items: Iterable[Installable]) -> bool + + * install_one(item: Installable) + * uninstall_one(item: Installable) + * is_installed_one(item: Installable) -> bool + } + + BaseInstaller <|-- SlurmInstaller + BaseInstaller <|-- StandaloneInstaller +``` diff --git a/doc/DEV.rst b/doc/DEV.rst deleted file mode 100644 index 298d47bc7..000000000 --- a/doc/DEV.rst +++ /dev/null @@ -1,98 +0,0 @@ -Development -=========== - -This chapter targets developers who want to contribute to the project's core. - -.. mermaid:: - - graph TD - subgraph _core - base_modules - core_implementations - registry - end - - subgraph runners - SlurmRunner - StandaloneRunner - end - - subgraph installers - SlurmInstaller - StandaloneInstaller - end - - subgraph systems - SlurmSystem - StandaloneSystem - end - - installers --> _core - runners --> _core - systems --> _core - -Core Modules ------------- - -We use `import-linter `_ to ensure no core modules import higher level modules. - -``Registry`` object is a singleton that holds implementation mappings. Users can register their own implementations to the registry or replace the default implementations. - -Cache ------ - -Some prerequisites can be installed. For example: - -Docker images, git repos with executable scripts, etc. All such "installables" are kept under the system's ``install_path``. - -Installables are shared among all tests. Therefore, if any number of tests use the same installable, it is installed only once for a particular system TOML. - -``PythonExecutable`` installs a Python project into its own virtual environment. When the selected -``pyproject.toml`` has an adjacent ``uv.lock``, it uses bundled uv to install the locked base dependencies -and the project, without extras or dependency groups. An outdated lockfile fails installation instead -of being updated. Projects without a lockfile continue to use pip. When ``dependencies_from_pyproject=False`` -selects an existing ``requirements.txt``, that selection takes precedence over the lockfile. - -.. mermaid:: - - classDiagram - class Installable { - <> - + __eq__(other: object) - + __hash__() - } - - class DockerImage { - + url: str - + install_path: str | Path - } - - class GitRepo { - + git_url: str - + commit_hash: str - + install_path: Path - } - - class PythonExecutable { - + git_repo: GitRepo - + venv_path: Path - } - - Installable <|-- DockerImage - Installable <|-- GitRepo - Installable <|-- PythonExecutable - PythonExecutable --> GitRepo - - class BaseInstaller { - <> - + install(items: Iterable[Installable]) - + uninstall(items: Iterable[Installable]) - + is_installed(items: Iterable[Installable]) -> bool - - * install_one(item: Installable) - * uninstall_one(item: Installable) - * is_installed_one(item: Installable) -> bool - } - - BaseInstaller <|-- SlurmInstaller - BaseInstaller <|-- StandaloneInstaller diff --git a/doc/Makefile b/doc/Makefile index a7bd7fb41..4e4574d84 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,59 +1,24 @@ # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -SPHINXOPTS ?= -SOURCEDIR = . -BUILDDIR = _build -BUILDDIR_CMS = _build-cms +# Set the provisioned instance URL in the publishing environment before deploying. +export FERN_DOCS_URL ?= cloudai-local-test.docs.buildwithfern.com -NODE_MODULES = $(BUILDDIR_CMS)/node_modules -MMDC = $(NODE_MODULES)/.bin/mmdc -PUPPETEER_CFG = $(BUILDDIR_CMS)/puppeteer-config.json - -.PHONY: help html cms-html install-mermaid clean +.PHONY: help reference check preview help: - @echo "Available targets:" - @echo " html - Build documentation for GitHub Pages (nvidia_sphinx_theme) -> $(BUILDDIR)/" - @echo " cms-html - Build documentation for BrightSpot CMS (sphinx_rtd_theme, PNG mermaid) -> $(BUILDDIR_CMS)/" - @echo " clean - Remove all build directories" - -# Default html target (GitHub Pages) -html: - @echo "Setting up environment for GitHub Pages build..." - @cd .. && uv sync --extra docs - @echo "Building documentation..." - @cd .. && uv run sphinx-build -M html "doc" "doc/$(BUILDDIR)" $(SPHINXOPTS) - @touch $(BUILDDIR)/html/.nojekyll - -cms-html: install-mermaid - @echo "Setting up environment for CMS build..." - @cd .. && uv sync --extra docs-cms - @echo "Building documentation..." - @cd .. && CLOUDAI_DOC_TARGET=cms uv run sphinx-build -M html "doc" "doc/$(BUILDDIR_CMS)" $(SPHINXOPTS) + @echo "reference - Generate Python reference snippets" + @echo "check - Generate references and validate Fern documentation" + @echo "preview - Generate references and start the local Fern server" -install-mermaid: $(MMDC) +reference: + cd .. && uv run --locked --extra docs python doc/generate_reference.py -$(MMDC): - @command -v npm >/dev/null 2>&1 || { echo "Error: npm is required for CMS build but not found. Please install Node.js/npm first."; exit 1; } - @echo "Installing @mermaid-js/mermaid-cli via npm..." - @mkdir -p $(BUILDDIR_CMS) - @cd $(BUILDDIR_CMS) && npm install @mermaid-js/mermaid-cli@^11 - @echo '{"args": ["--no-sandbox"]}' > $(PUPPETEER_CFG) - @echo "mermaid-cli installed successfully." +check: reference + cd .. && npm --prefix fern ci + cd .. && npm --prefix fern run check -clean: - @rm -rf $(BUILDDIR) $(BUILDDIR_CMS) +preview: reference + cd .. && npm --prefix fern ci + cd .. && npm --prefix fern run dev diff --git a/doc/Tutorial.mdx b/doc/Tutorial.mdx new file mode 100644 index 000000000..64c225d63 --- /dev/null +++ b/doc/Tutorial.mdx @@ -0,0 +1,272 @@ + + +This chapter outlines a tutorial on how to utilize the CloudAI framework. Please follow the steps in the same sequence to ensure successful execution: + +- [Step 1: Creating a Docker Image](Tutorial.mdx#step-1-creating-docker-image) +- [Step 2: Preparing Configuration Files](Tutorial.mdx#step-2-preparing-configuration-files) +- [Step 3: Testing Definition](Tutorial.mdx#step-3-testing-definition) +- [Step 4: System Configuration](Tutorial.mdx#step-4-system-configuration) +- [Step 5: Testing Configuration](Tutorial.mdx#step-5-testing-configuration) +- [Step 6: Running Experiments](Tutorial.mdx#step-6-running-experiments) +- [Step 7: Generating Reports](Tutorial.mdx#step-7-generating-reports) +- [Test in Scenario](Tutorial.mdx#test-in-scenario) + + + +## Creating a Docker Image + +To create a Docker image, follow these steps: + +1. **Set Up the GitLab Repository:** Start by setting up a repository on GitLab to host your docker image. For this example, use `gitlab-url.com/cloudai/nccl-test`. + +2. **Write the Dockerfile:** + + ``` dockerfile + FROM nvcr.io/nvidia/pytorch:24.02-py3 + ``` + +3. **Build and Push the Docker Image:** Build the docker image with the Dockerfile and upload it to the designated repository: + + ``` bash + docker build -t gitlab-url.com/cloudai/nccl-test . + docker push gitlab-url.com/cloudai/nccl-test + ``` + +4. **Verify the Docker Image:** Test the docker image by running it with `srun` to verify that the docker image runs correctly: + + ``` bash + srun \ + --mpi=pmix \ + --container-image=gitlab-url.com/cloudai/nccl-test \ + all_reduce_perf_mpi \ + --nthreads 1 \ + --ngpus 1 \ + --minbytes 128 \ + --maxbytes 16G \ + --stepbytes 1M \ + --op sum \ + --datatype float \ + --root 0 \ + --iters 100 \ + --warmup_iters 50 \ + --agg_iters 1 \ + --average 1 \ + --parallel_init 0 \ + --check 1 \ + --blocking 0 \ + --cudagraph 0 \ + --stepfactor 2 + ``` + + + +## Preparing Configuration Files + +CloudAI is fully configurable via a set of TOML configuration files. You can find examples of these files under `conf/common`. In this guide, we will use the following configuration files: + +1. `CONFIGS_DIR/system.toml` - Describes the system configuration. +2. `CONFIGS_DIR/tests/nccl_test.toml` - Describes the test to run. +3. `CONFIGS_DIR/scenario.toml` - Describes the test scenario configuration. + + + +## Testing Definition + +Test definition is a Pydantic model that describes the arguments of a test. Such models should be inherited from the `TestDefinition` class: + +``` python +class MyTestCmdArgs(CmdArgs): + an_arg: str | list[str] + docker_image_url: str = "nvcr.io/nvidia/pytorch:24.02-py3" + +class MyTestDefinition(TestDefinition): + cmd_args: MyTestCmdArgs +``` + +Notice that `cmd_args.docker_image_url` uses `nvcr.io/nvidia/pytorch:24.02-py3`, but you can use the Docker image from Step 1. + +`an_arg` has a mixed type of `str | list[str]`, so in a TOML config it can be defined as either: + +``` toml +an_arg = "a single string" +``` + +Or + +``` toml +an_arg = ["list", "of", "strings"] +``` + +When a list is used, CloudAI will automatically generate multiple test cases for each value in the list. + +A custom test definition should be registered to handle relevant Test Configs. For this, `Registry()` object is used: + +``` python +Registry().add_test_definition("MyTest", MyTestDefinition) +Registry().add_test_template("MyTest", MyTest) +``` + +Relevant Test Configurations should specify `test_template_name = MyTest` to use the custom test definition. + + + +## System Configuration + +System configuration describes how the system configuration works. You can find more examples of system configuration under `conf/common/system/`. The example below is for demonstration purposes. The following is the `CONFIGS_DIR/system.toml` file: + +``` toml +name = "my-cluster" +scheduler = "slurm" + +install_path = "./install" +output_path = "./results" +cache_docker_images_locally = true +default_partition = "" + +mpi = "pmix" +gpus_per_node = 8 +ntasks_per_node = 8 + +[[partitions]] +name = "partition_1" +``` + +Replace `` with the name of the partition you want to use. You can find the partition name by running `sinfo` on the cluster. + + + +## Testing Configuration + +Test configuration describes a particular test configuration to be run. It is based on test definition and will be used in a test scenario. Below is the `CONFIGS_DIR/tests/nccl_test.toml` file, definition is based on the built-in `NcclTest` definition: + +``` toml +name = "nccl_test_all_reduce_single_node" +description = "all_reduce" +test_template_name = "NcclTest" + +[cmd_args] +subtest_name = "all_reduce_perf_mpi" +ngpus = 1 +minbytes = "8M" +maxbytes = "16G" +iters = 5 +warmup_iters = 3 +stepfactor = 2 +``` + +You can find more examples under `conf/common/test`. In a test schema file, you can adjust arguments as shown above. In the `cmd_args` section, you can provide different values other than the default values for each argument. In `extra_cmd_args`, you can provide additional arguments that will be appended after the NCCL test command. You can specify additional environment variables in the `extra_env_vars` section. + + + +## Running Experiments + +Test Scenario uses test description from Step 5. Below is the `CONFIGS_DIR/scenario.toml` file: + +``` toml +name = "nccl-test" + +[[Tests]] +id = "allreduce.1" +num_nodes = 1 +test_name = "nccl_test_all_reduce_single_node" +time_limit = "00:20:00" + +[[Tests]] +id = "allreduce.2" +num_nodes = 1 +test_name = "nccl_test_all_reduce_single_node" +time_limit = "00:20:00" + [[Tests.dependencies]] + type = "start_post_comp" + id = "allreduce.1" +``` + +Notes on the test scenario: + +1. `id` is a mandatory field and must be unique for each test. +2. The `test_name` specifies the test definition from one of the Test TOML files. Node lists and time limits are optional. +3. All workload definition parameters can be overridden in a test case, including workload-specific sections such as `semantic_eval_cmd_args`. The referenced test's `test_template_name` cannot be changed. +4. If needed, `nodes` should be described as a list of node names as shown in a Slurm system. Alternatively, if groups are defined in the system schema, you can ask CloudAI to allocate a specific number of nodes from a specified partition and group. For example, `nodes = ['PARTITION:GROUP:16']` allocates 16 nodes from group `GROUP` and partition `PARTITION`. +5. There are three types of dependencies: `start_post_comp`, `start_post_init` and `end_post_comp`. + - `start_post_comp` means that the current test should be started after a specific delay of the completion of the depending test. + - `start_post_init` means that the current test should start after the start of the depending test. + - `end_post_comp` means that the current test should be completed after the completion of the depending test. + +All dependencies are described as a depending test name. The name should be taken from the test name as set in the test scenario. + +To generate NCCL test commands without actual execution, use the `dry-run` mode. You can review `debug.log` (or other file specified with `--log-file`) to see the generated commands from CloudAI. Please note that group node allocations are not currently supported in the `dry-run` mode. + +``` bash +cloudai dry-run \ + --test-scenario CONFIGS_DIR/scenario.toml \ + --system-config CONFIGS_DIR/system.toml \ + --tests-dir CONFIGS_DIR/tests/ +``` + +You can run NCCL test experiments with the following command. Whenever you run CloudAI in the `run` mode, a new directory will be created under the results directory with the timestamp. In the directory, you can find the results from the test scenario including stdout and stderr. Once completed successfully, you can find generated reports under the directories as well. + +``` bash +cloudai run \ + --test-scenario CONFIGS_DIR/scenario.toml \ + --system-config CONFIGS_DIR/system.toml \ + --tests-dir CONFIGS_DIR/tests/ +``` + + + +## Generating Reports + +Once the test scenario is completed, it is possible to generate reports using the following command: + +``` bash +cloudai generate-report \ + --test-scenario CONFIGS_DIR/scenario.toml \ + --system-config CONFIGS_DIR/system.toml \ + --tests-dir CONFIGS_DIR/tests/ \ + --result-dir results/2024-06-18_17-40-13/ +``` + +`--result-dir` accepts one scenario run result directory. + + + +## Test in Scenario + +It is possible to override some args or even fully define a workload inside a scenario file: + +``` toml +name = "nccl-test" + +[[Tests]] +id = "allreduce.in.scenario" +num_nodes = 1 +time_limit = "00:20:00" + +name = "nccl_test_all_reduce_single_node" +description = "all_reduce" +test_template_name = "NcclTest" + + [Tests.cmd_args] + subtest_name = "all_reduce_perf_mpi" + ngpus = 1 + minbytes = "8M" + maxbytes = "16G" + iters = 5 + warmup_iters = 3 + stepfactor = 2 + +[[Tests]] +id = "allreduce.override" +num_nodes = 1 +test_name = "nccl_test_all_reduce_single_node" +time_limit = "00:20:00" + + [Tests.cmd_args] + stepfactor = 4 +``` + +`allreduce.in.scenario` fully defines a workload; in this case `test_name` must not be set, while `name`, `description` and `test_template_name` must be set. + +`allreduce.override` overrides only `stepfactor` arg from the test defined in the tests directory. + +If a scenario contains only fully defined tests, `--tests-dir` arg is not required. diff --git a/doc/Tutorial.rst b/doc/Tutorial.rst deleted file mode 100644 index f635c9299..000000000 --- a/doc/Tutorial.rst +++ /dev/null @@ -1,283 +0,0 @@ -Tutorial -======== - -This chapter outlines a tutorial on how to utilize the CloudAI framework. Please follow the steps in the same sequence to ensure successful execution: - -- :ref:`Step 1: Creating a Docker Image ` -- :ref:`Step 2: Preparing Configuration Files ` -- :ref:`Step 3: Testing Definition ` -- :ref:`Step 4: System Configuration ` -- :ref:`Step 5: Testing Configuration ` -- :ref:`Step 6: Running Experiments ` -- :ref:`Step 7: Generating Reports ` -- :ref:`Test in Scenario ` - -.. _step-1-creating-docker-image: - -Creating a Docker Image -~~~~~~~~~~~~~~~~~~~~~~~ - -To create a Docker image, follow these steps: - -#. **Set Up the GitLab Repository:** Start by setting up a repository on GitLab to host your docker image. For this example, use ``gitlab-url.com/cloudai/nccl-test``. - -#. **Write the Dockerfile:** - - .. code-block:: dockerfile - - FROM nvcr.io/nvidia/pytorch:24.02-py3 - -#. **Build and Push the Docker Image:** Build the docker image with the Dockerfile and upload it to the designated repository: - - .. code-block:: bash - - docker build -t gitlab-url.com/cloudai/nccl-test . - docker push gitlab-url.com/cloudai/nccl-test - -#. **Verify the Docker Image:** Test the docker image by running it with ``srun`` to verify that the docker image runs correctly: - - .. code-block:: bash - - srun \ - --mpi=pmix \ - --container-image=gitlab-url.com/cloudai/nccl-test \ - all_reduce_perf_mpi \ - --nthreads 1 \ - --ngpus 1 \ - --minbytes 128 \ - --maxbytes 16G \ - --stepbytes 1M \ - --op sum \ - --datatype float \ - --root 0 \ - --iters 100 \ - --warmup_iters 50 \ - --agg_iters 1 \ - --average 1 \ - --parallel_init 0 \ - --check 1 \ - --blocking 0 \ - --cudagraph 0 \ - --stepfactor 2 - -.. _step-2-preparing-configuration-files: - -Preparing Configuration Files -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -CloudAI is fully configurable via a set of TOML configuration files. You can find examples of these files under ``conf/common``. In this guide, we will use the following configuration files: - -#. ``CONFIGS_DIR/system.toml`` - Describes the system configuration. -#. ``CONFIGS_DIR/tests/nccl_test.toml`` - Describes the test to run. -#. ``CONFIGS_DIR/scenario.toml`` - Describes the test scenario configuration. - -.. _step-3-testing-definition: - -Testing Definition -~~~~~~~~~~~~~~~~~~ - -Test definition is a Pydantic model that describes the arguments of a test. Such models should be inherited from the ``TestDefinition`` class: - -.. code-block:: python - - class MyTestCmdArgs(CmdArgs): - an_arg: str | list[str] - docker_image_url: str = "nvcr.io/nvidia/pytorch:24.02-py3" - - class MyTestDefinition(TestDefinition): - cmd_args: MyTestCmdArgs - -Notice that ``cmd_args.docker_image_url`` uses ``nvcr.io/nvidia/pytorch:24.02-py3``, but you can use the Docker image from Step 1. - -``an_arg`` has a mixed type of ``str | list[str]``, so in a TOML config it can be defined as either: - -.. code-block:: toml - - an_arg = "a single string" - -Or - -.. code-block:: toml - - an_arg = ["list", "of", "strings"] - -When a list is used, CloudAI will automatically generate multiple test cases for each value in the list. - -A custom test definition should be registered to handle relevant Test Configs. For this, ``Registry()`` object is used: - -.. code-block:: python - - Registry().add_test_definition("MyTest", MyTestDefinition) - Registry().add_test_template("MyTest", MyTest) - -Relevant Test Configurations should specify ``test_template_name = MyTest`` to use the custom test definition. - -.. _step-4-system-configuration: - -System Configuration -~~~~~~~~~~~~~~~~~~~~ - -System configuration describes how the system configuration works. You can find more examples of system configuration under ``conf/common/system/``. The example below is for demonstration purposes. The following is the ``CONFIGS_DIR/system.toml`` file: - -.. code-block:: toml - - name = "my-cluster" - scheduler = "slurm" - - install_path = "./install" - output_path = "./results" - cache_docker_images_locally = true - default_partition = "" - - mpi = "pmix" - gpus_per_node = 8 - ntasks_per_node = 8 - - [[partitions]] - name = "partition_1" - -Replace ```` with the name of the partition you want to use. You can find the partition name by running ``sinfo`` on the cluster. - -.. _step-5-testing-configuration: - -Testing Configuration -~~~~~~~~~~~~~~~~~~~~~ - -Test configuration describes a particular test configuration to be run. It is based on test definition and will be used in a test scenario. Below is the ``CONFIGS_DIR/tests/nccl_test.toml`` file, definition is based on the built-in ``NcclTest`` definition: - -.. code-block:: toml - - name = "nccl_test_all_reduce_single_node" - description = "all_reduce" - test_template_name = "NcclTest" - - [cmd_args] - subtest_name = "all_reduce_perf_mpi" - ngpus = 1 - minbytes = "8M" - maxbytes = "16G" - iters = 5 - warmup_iters = 3 - stepfactor = 2 - -You can find more examples under ``conf/common/test``. In a test schema file, you can adjust arguments as shown above. In the ``cmd_args`` section, you can provide different values other than the default values for each argument. In ``extra_cmd_args``, you can provide additional arguments that will be appended after the NCCL test command. You can specify additional environment variables in the ``extra_env_vars`` section. - -.. _step-6-running-experiments: - -Running Experiments -~~~~~~~~~~~~~~~~~~~ - -Test Scenario uses test description from Step 5. Below is the ``CONFIGS_DIR/scenario.toml`` file: - -.. code-block:: toml - - name = "nccl-test" - - [[Tests]] - id = "allreduce.1" - num_nodes = 1 - test_name = "nccl_test_all_reduce_single_node" - time_limit = "00:20:00" - - [[Tests]] - id = "allreduce.2" - num_nodes = 1 - test_name = "nccl_test_all_reduce_single_node" - time_limit = "00:20:00" - [[Tests.dependencies]] - type = "start_post_comp" - id = "allreduce.1" - -Notes on the test scenario: - -#. ``id`` is a mandatory field and must be unique for each test. -#. The ``test_name`` specifies the test definition from one of the Test TOML files. Node lists and time limits are optional. -#. All workload definition parameters can be overridden in a test case, including workload-specific sections such as ``semantic_eval_cmd_args``. The referenced test's ``test_template_name`` cannot be changed. -#. If needed, ``nodes`` should be described as a list of node names as shown in a Slurm system. Alternatively, if groups are defined in the system schema, you can ask CloudAI to allocate a specific number of nodes from a specified partition and group. For example, ``nodes = ['PARTITION:GROUP:16']`` allocates 16 nodes from group ``GROUP`` and partition ``PARTITION``. -#. There are three types of dependencies: ``start_post_comp``, ``start_post_init`` and ``end_post_comp``. - - - ``start_post_comp`` means that the current test should be started after a specific delay of the completion of the depending test. - - ``start_post_init`` means that the current test should start after the start of the depending test. - - ``end_post_comp`` means that the current test should be completed after the completion of the depending test. - -All dependencies are described as a depending test name. The name should be taken from the test name as set in the test scenario. - -To generate NCCL test commands without actual execution, use the ``dry-run`` mode. You can review ``debug.log`` (or other file specified with ``--log-file``) to see the generated commands from CloudAI. Please note that group node allocations are not currently supported in the ``dry-run`` mode. - -.. code-block:: bash - - cloudai dry-run \ - --test-scenario CONFIGS_DIR/scenario.toml \ - --system-config CONFIGS_DIR/system.toml \ - --tests-dir CONFIGS_DIR/tests/ - -You can run NCCL test experiments with the following command. Whenever you run CloudAI in the ``run`` mode, a new directory will be created under the results directory with the timestamp. In the directory, you can find the results from the test scenario including stdout and stderr. Once completed successfully, you can find generated reports under the directories as well. - -.. code-block:: bash - - cloudai run \ - --test-scenario CONFIGS_DIR/scenario.toml \ - --system-config CONFIGS_DIR/system.toml \ - --tests-dir CONFIGS_DIR/tests/ - - -.. _step-7-generating-reports: - -Generating Reports -~~~~~~~~~~~~~~~~~~ - -Once the test scenario is completed, it is possible to generate reports using the following command: - -.. code-block:: bash - - cloudai generate-report \ - --test-scenario CONFIGS_DIR/scenario.toml \ - --system-config CONFIGS_DIR/system.toml \ - --tests-dir CONFIGS_DIR/tests/ \ - --result-dir results/2024-06-18_17-40-13/ - -``--result-dir`` accepts one scenario run result directory. - -.. _test-in-scenario: - -Test in Scenario -~~~~~~~~~~~~~~~~~~~ - -It is possible to override some args or even fully define a workload inside a scenario file: - -.. code-block:: toml - - name = "nccl-test" - - [[Tests]] - id = "allreduce.in.scenario" - num_nodes = 1 - time_limit = "00:20:00" - - name = "nccl_test_all_reduce_single_node" - description = "all_reduce" - test_template_name = "NcclTest" - - [Tests.cmd_args] - subtest_name = "all_reduce_perf_mpi" - ngpus = 1 - minbytes = "8M" - maxbytes = "16G" - iters = 5 - warmup_iters = 3 - stepfactor = 2 - - [[Tests]] - id = "allreduce.override" - num_nodes = 1 - test_name = "nccl_test_all_reduce_single_node" - time_limit = "00:20:00" - - [Tests.cmd_args] - stepfactor = 4 - -``allreduce.in.scenario`` fully defines a workload; in this case ``test_name`` must not be set, while ``name``, ``description`` and ``test_template_name`` must be set. - -``allreduce.override`` overrides only ``stepfactor`` arg from the test defined in the tests directory. - -If a scenario contains only fully defined tests, ``--tests-dir`` arg is not required. diff --git a/doc/USER_GUIDE.mdx b/doc/USER_GUIDE.mdx new file mode 100644 index 000000000..e1fd00836 --- /dev/null +++ b/doc/USER_GUIDE.mdx @@ -0,0 +1,580 @@ + + +The purpose of this chapter is to help users utilize and understand the different components of the CloudAI framework. The chapter covers topics such as adding new tests and downloading datasets for running NeMo-launcher. + + + +## System Schema + +In this section, we introduce the concept of the system schema, explain the meaning of each field, and describe how the fields should be used. The system schema is a TOML file that allows users to define a system's configuration. + +``` toml +name = "example-cluster" +scheduler = "slurm" + +install_path = "./install" +output_path = "./results" +default_partition = "partition_1" + +mpi = "pmix" +gpus_per_node = 8 +ntasks_per_node = 8 + +cache_docker_images_locally = true + +[[partitions]] +name = "partition_1" + + [[partitions.groups]] + name = "group_a" + nodes = ["node-[001-025]"] + + [[partitions.groups]] + name = "group_b" + nodes = ["node-[026-050]"] + +[[partitions]] +name = "partition_2" + +[global_env_vars] +# NCCL Specific Configurations +NCCL_IB_GID_INDEX = "3" +NCCL_IB_TIMEOUT = "20" +NCCL_IB_QPS_PER_CONNECTION = "4" + +# Device Visibility Configuration +MELLANOX_VISIBLE_DEVICES = "0,3,4,5,6,9,10,11" +CUDA_VISIBLE_DEVICES = "0,1,2,3,4,5,6,7" +``` + + + +## Field Descriptions + +| **Field** | **Description** | +|----|----| +| **name** | Specifies the name of the system. Users can choose any name that is convenient for them. | +| **scheduler** | Indicates the type of system. It should be one of the supported types, currently `slurm` or `standalone`. `slurm` refers to a system with the Slurm scheduler, while `standalone` refers to a single-node system without any slave nodes. Other values are possible depending on the available schedulers supported by CloudAI. | +| **install_path** | Specifies the path where test prerequisites are installed. Docker images are downloaded to this path if the user chooses to cache Docker images. Relative paths are converted to absolute paths using the current working directory when the system is loaded, preserving symbolic links. | +| **output_path** | Defines the default path where outputs are stored. Whenever a user runs a test scenario, a new subdirectory will be created under this path. | +| **default_partition** | Specifies the default partition where jobs are scheduled. | +| **partitions** | Describes the available partitions and nodes within those partitions. | +| **\[optional\] groups** | Within the same partition, users can define groups of nodes. The group concept can be used to allocate nodes from specific groups in a test scenario schema. For instance, this feature is useful for specifying topology awareness. Groups represent logical partitioning of nodes and users are responsible for ensuring no overlap across groups. | +| **mpi** | Indicates the Process Management Interface (PMI) implementation to be used for inter-process communication. | +| **gpus_per_node** and **ntasks_per_node** | These are Slurm arguments passed to the `sbatch` script and `srun`. | +| **cache_docker_images_locally** | Specifies whether CloudAI should cache remote Docker images locally during installation. If set to `true`, CloudAI will cache the Docker images, enabling local access without needing to download them each time a test is run. This approach saves network bandwidth but requires more disk capacity. If set to `false`, CloudAI will allow Slurm to download the Docker images as needed when they are not cached locally by Slurm. | +| **global_env_vars** | Lists all global environment variables that will be applied globally whenever tests are run. | +| **\[optional\] slurm_api** | Uses Slurm REST API instead of local Slurm CLI tools. Set `url` and optional `verify_certs` and `headers`. Header values may reference environment variables using `${NAME}`. | + +Field Descriptions + + + +### Slurm REST API + +Set `slurm_api.version` to match your service: `v0.0.38` (the default), `v0.0.40`, `v0.0.41`, or `v0.0.42`. The service must enable the `slurm` endpoint backed by `slurmctld`. To retrieve status and timing after a job leaves the controller's records, it must also expose the `slurmdb` endpoint backed by `slurmdbd`. + +Workload steps inherit the system's `ntasks_per_node` and `gpus_per_node` settings in REST mode, as they do with `sbatch`. Explicit `srun` options can override these defaults for individual steps. + +``` toml +[slurm_api] +url = "https://slurm-api.example.com" +version = "v0.0.42" +verify_certs = true + + [slurm_api.headers] + X-SLURM-USER-NAME = "${SLURM_USER}" + X-SLURM-USER-TOKEN = "${SLURM_JWT}" +``` + +Workloads that invoke their own Slurm launcher instead of producing an sbatch script are not supported in REST mode. + + + +## RunAI Scheduler + +When using RunAI as the scheduler, additional fields must be specified in the system schema TOML file. The following is a list of required fields and how to set them: + +``` toml +name = "runai-cluster" +scheduler = "runai" + +install_path = "./install" +output_path = "./results" + +base_url = "http://runai.example.com" # The URL of your RunAI system, typically the same as used for the web interface. +verify_certs = true # Applies to HTTPS requests and training log connections. +user_email = "your_email" # The email address used to log into the RunAI system. +app_id = "your_app_id" # Obtained by creating an application in the RunAI web interface. +app_secret = "" # Fill in the secret obtained with the app_id. +project_id = "your_project_id" # Project ID assigned or created in the RunAI system (usually an integer). +cluster_id = "your_cluster_id" # Cluster ID in UUID format (e.g., a69928cc-ccaa-48be-bda9-482440f4d855). +``` + +- After logging into the RunAI web interface, navigate to Access → Applications and create a new application to obtain `app_id` and `app_secret` +- Use your assigned project and cluster IDs. Contact your administrator if they are not available +- All other fields follow the same semantics as in the Slurm system schema (e.g., `install_path`, `output_path`) + +RunAI HTTP requests use a 30-second timeout. The optional `verify_certs` setting defaults to `true` and controls server TLS certificate verification for both HTTPS requests and training log connections. Set it to `false` to disable verification. For training log connections with a private certificate authority, install its CA certificate in the system trust store. + + + +## Test Scenario Schema + +A test scenario is a set of tests with specific dependencies between them. A test scenario is described in a TOML schema file. The following is an example of a test scenario file: + +``` toml +name = "nccl-test" + +[[Tests]] +id = "Tests.1" +test_name = "nccl_test_all_reduce" +num_nodes = "2" +time_limit = "00:20:00" + +[[Tests]] +id = "Tests.2" +test_name = "nccl_test_all_gather" +num_nodes = "2" +time_limit = "00:20:00" + [[Tests.dependencies]] + type = "start_post_comp" + id = "Tests.1" + +[[Tests]] +id = "Tests.3" +test_name = "nccl_test_reduce_scatter" +num_nodes = "2" +time_limit = "00:20:00" + [[Tests.dependencies]] + type = "start_post_comp" + id = "Tests.2" +``` + +The `name` field is the test scenario name, which can be any unique identifier for the scenario. Each test has a section name, following the convention `Tests.1`, `Tests.2`, etc., with an increasing index. The `name` of a test should be specified in this section and must correspond to an entry in the test schema. If a test in a test scenario is not present in the test schema, CloudAI will not be able to identify it. + +There are two ways to specify nodes: + +- Using the `num_nodes` field as shown in the example +- Specifying nodes explicitly like `nodes = ["node-001", "node-002"]` + +**Note:** When an explicit node list is provided (e.g., `nodes = ["node-001", "node-002"]`), CloudAI lets Slurm apply the arbitrary distribution policy for task placement. + +Alternatively, you can utilize the groups feature in the system schema to specify nodes like `nodes = ['PARTITION_NAME:GROUP_NAME:NUM_NODES']`, which allocates `num_nodes` from the group name in the specified partition. You can also use `nodes = ['PARTITION_NAME:GROUP_NAME:max_avail']`, which allocates all the available nodes from the group name in the specified partition. + +You can optionally specify a time limit in the Slurm format. Tests can have dependencies. If no dependencies are specified, all tests will run in parallel. + +CloudAI supports three types of dependencies: + +- `start_post_init` +- `start_post_comp` +- `end_post_comp` + +Dependencies of a test can be described as a subsection of the test. It requires other tests' `id` and dependency `type`. + +- `start_post_init` means the test starts after the prior test begins, with a specified delay. +- `start_post_comp` means the test starts after the prior test completes. +- `end_post_comp` means the test ends when the prior test completes. + + + +## Agents Configuration + +For DSE workloads, you can pass agent configuration via `agent_config` in the scenario. + +`BaseAgentConfig` includes: + +- `random_seed`: controls deterministic random behavior in agents. +- `start_action`: controls the very first action strategy (`"random"` or `"first"`). +- `rewards` (optional): nested table mapped to `RewardOverrides`. When present, `CloudAIGymEnv` uses it for constraint failures and for substituting failed-metric values in observations. + - `constraint_failure`: reward returned when `TestDefinition.constraint_check` fails on a step. If omitted or unset, the environment uses `-1.0`. + - `metric_failure`: value written into the observation vector when a metric is missing or is the canonical error sentinel `METRIC_ERROR` (a distinct object from numeric metrics, so a valid `-1.0` result is not mistaken for an error). If omitted or unset, observations use `-1.0` for that slot. + +Example: + +``` toml +[[Tests]] +id = "Tests.1" +test_name = "nccl_test_all_reduce" +agent = "grid_search" +agent_steps = 10 +agent_metrics = ["default"] +agent_reward_function = "inverse" + + [Tests.agent_config] + random_seed = 123 + start_action = "first" + + [Tests.agent_config.rewards] + constraint_failure = -5.0 + metric_failure = 0.0 +``` + +When an agent honors `start_action = "first"`, it should start from `CloudAIGymEnv.first_sweep` (the sweep is built from first values of each sweep parameter). `start_action = "random"` means starting from a random action, typically seeded by `random_seed`. + +Custom agents may extend the `BaseAgentConfig` and offer more parameters to configure. + + + +### DSE Parameter Exclusions + +CloudAI builds the DSE parameter space implicitly from list-valued fields under `cmd_args`, list-valued `extra_env_vars`, and list-valued `num_nodes`. Most lists mean "try each value", but some workload settings are real list-valued configuration, such as worker port lists or ordered benchmark phases. + +Use `dse_excluded_args` when a list under `cmd_args` should stay intact instead of becoming a sweep dimension. Entries must be dot-separated paths that start with `cmd_args.` and may point to either a single field or a parent object. Matching is prefix-based, so excluding `cmd_args.foo` also excludes nested list-valued fields such as `cmd_args.foo.bar` from DSE parameter discovery. + +``` toml +[[Tests]] +id = "Tests.1" +test_name = "my_test" +dse_excluded_args = ["cmd_args.lmcache.lmcache_worker_ports"] + + [Tests.cmd_args.lmcache] + chunk_size = [256, 512] + lmcache_worker_ports = [8788, 8789, 8790, 8791] +``` + +In this example, `cmd_args.lmcache.chunk_size` is still swept, while `cmd_args.lmcache.lmcache_worker_ports` is passed through as one list value. The exclusion does not remove or mutate the field; it only prevents CloudAI from adding that path to the DSE parameter space. + +`dse_excluded_args` currently applies only to `cmd_args` paths. It does not exclude list-valued `extra_env_vars` or `num_nodes`; those lists are still interpreted as sweep dimensions. To exclude many nested list fields at once, exclude their common parent path. Common examples are `cmd_args.aiperf_phases` and `cmd_args.lmcache.lmcache_worker_ports`. + + + +### Metric Errors and Report Strategies + +Report generation strategies signal a failed or missing metric by returning the singleton `METRIC_ERROR` from `cloudai.core` (type `MetricErrorSentinel`; `float(METRIC_ERROR)` is `-1.0` for numeric use). The declared return type of `ReportGenerationStrategy.get_metric` is `MetricValue` (`float | MetricErrorSentinel`). + +When branching on success vs failure, use **identity** (`value is METRIC_ERROR`), not equality to `-1.0`, so a legitimate metric of `-1.0` is not treated as an error. The gym observation path uses that identity check before applying `agent_config.rewards.metric_failure`. + + + +## Configuring HTTP Data Repository + +The HTTP Data Repository is currently supported for Slurm systems only. To enable access, you must update your system schema file and create a credential file in your CloudAI project's root directory. + +The following steps are required to configure the HTTP Data Repository: + +- [Step 1: Update the System Schema File](USER_GUIDE.mdx#step-1-update-system-schema-file) +- [Step 2: Create the Credential File](USER_GUIDE.mdx#step-2-create-credential-file) +- [Step 3: Usage](USER_GUIDE.mdx#step-3-usage) + + + +### Updating the System Schema File + +Add the following section to your system schema TOML file (e.g., `system_schema.toml`): + +``` toml +[data_repository] +endpoint = "https://my-data-endpoint.com" +``` + +Replace the endpoint with your actual data repository URL. + + + +### Creating the Credential File + +In the root of your CloudAI project (i.e., the current working directory), create a file named `.cloudai.toml` with the following content: + +``` toml +[data_repository] +token = "" +``` + +Replace `` with your actual token. + + + +### Usage + +Both the endpoint and token must be valid for the HTTP Data Repository to function correctly. If either is missing or incorrect, data will not be posted. + + + +## Using Test Hooks in CloudAI + +A test hook in CloudAI is a specialized test that runs either before or after each main test in a scenario, providing flexibility to prepare the environment or clean up resources. Hooks are defined as pre-test or post-test and referenced in the test scenario’s TOML file using `pre_test` and `post_test` fields. + +``` toml +name = "nccl-test" + +pre_test = "nccl_test_pre" +post_test = "nccl_test_post" + +[[Tests]] +id = "Tests.1" +test_name = "nccl_test_all_reduce" +time_limit = "00:20:00" +``` + +CloudAI organizes hooks in a dedicated directory structure: + +- Hook directory: All hook configurations reside in `conf/hook/` +- Hook test scenarios: Place pre-test hook and post-test hook scenario files in `conf/hook/` +- Hook tests: Place individual tests referenced in hooks within `conf/hook/test/` + +In the execution flow, pre-test hooks run before the main test, which only executes if the pre-test completes successfully. Post-test hooks follow the main test, provided the prior steps succeed. If a pre-test hook fails, the main test and its post-test hook are skipped. + +``` toml +name = "nccl_test_pre" + +[[Tests]] +id = "Tests.1" +test_name = "nccl_test_all_reduce" +time_limit = "00:20:00" +``` + + + +## Downloading DeepSeek Weights + +To run DeepSeek R1 tests in CloudAI, you must download the model weights in advance. These weights are distributed via the NVIDIA NGC Registry and must be manually downloaded using the NGC CLI. + +The following steps are required to download the DeepSeek weights: + +- [Step 1: Installing NGC CLI](USER_GUIDE.mdx#step-1-installing-ngc-cli) +- [Step 2: Configuring NGC CLI](USER_GUIDE.mdx#step-2-configuring-ngc-cli) +- [Step 3: Downloading the Weights](USER_GUIDE.mdx#step-3-downloading-the-weights) +- [Step 4: Verifying the Download](USER_GUIDE.mdx#step-4-verifying-the-download) + + + +### Installing NGC CLI + +Download and install the NGC CLI using the following commands: + +``` bash +wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/3.64.2/files/ngccli_linux.zip -O ngccli_linux.zip +unzip ngccli_linux.zip +chmod u+x ngc-cli/ngc +echo "export PATH=\"$PATH:$(pwd)/ngc-cli\"" >> ~/.bash_profile && source ~/.bash_profile +``` + +This will make the `ngc` command available in your terminal. + + + +### Configuring NGC CLI + +Authenticate your CLI with your NGC API key by running: + +``` bash +ngc config set +``` + +When prompted, paste your API key, which you can obtain from [https://org.ngc.nvidia.com/setup](https://org.ngc.nvidia.com/setup). + + + +### Downloading the Weights + +Navigate to the directory where you want the DeepSeek model weights to be stored, then run: + +``` bash +ngc registry model download-version nim/deepseek-ai/deepseek-r1-instruct:hf-5dde110-nim-fp8 --dest . +``` + +This command will create a folder named: + +``` text +deepseek-r1-instruct_vhf-5dde110-nim-fp8/ +``` + +inside your current directory. + + + +### Verifying the Download + +Make sure the full model has been downloaded by checking the folder size: + +``` bash +du -sh deepseek-r1-instruct_vhf-5dde110-nim-fp8 +``` + +The expected size is approximately 642 GB. If it’s significantly smaller, remove the folder and re-run the download. + + + +## Slurm + +The following are subtopics related to Slurm: + + + +### Single sbatch vs Per-case sbatch + +CloudAI supports two modes for submitting Slurm jobs: + +1. (default) Per-case sbatch mode: each case is submitted as a separate sbatch job, allowing flexible scheduling and dependency management. + - Suits best when cases can run in parallel or there are no dependencies between cases. +2. Single sbatch mode: all cases are submitted together in a single sbatch job and share the same nodes. Each next case starts after the previous one completes. To enable it one needs to pass `--single-sbatch` flag to `cloudai run` command (works only for Slurm systems). + - Suits best for jobs when cases need to run on the same nodes for performance reasons. + - There is no support for dependency management between cases yet; all jobs run one after another. + +Assuming two cases in a scenario like this: + +``` toml +[[Tests]] +id = "nccl.all_reduce" +test_name = "nccl-all_reduce" +num_nodes = 2 +time_limit = "00:20:00" + +[[Tests]] +id = "nccl.all_gather" +test_name = "nccl-all_gather" +num_nodes = 2 +time_limit = "00:20:00" +``` + +Regular output directory structure (some files are omitted for clarity): + +``` bash +$ tree results/scenario +results/scenario +├── nccl.all_gather +│ └── 0 +│ ├── cloudai_sbatch_script.sh +│ ├── env_vars.sh +│ ├── metadata/ +│ ├── stderr.txt +│ ├── stdout.txt +│ └── test-run.toml +└── nccl.all_reduce + └── 0 + ├── cloudai_sbatch_script.sh + ├── env_vars.sh + ├── metadata/ + ├── stderr.txt + ├── stdout.txt + └── test-run.toml +``` + +Single sbatch mode output directory structure: + +``` bash +$ tree results/scenario +results/scenario +├── cloudai_sbatch_script.sh +├── common.err +├── common.out +├── metadata/ +├── nccl.all_gather +│ └── 0 +│ ├── env_vars.sh +│ ├── stderr.txt +│ ├── stdout.txt +│ └── test-run.toml +├── nccl.all_reduce +│ └── 0 +│ ├── env_vars.sh +│ ├── stderr.txt +│ ├── stdout.txt +│ └── test-run.toml +└── slurm-job.toml +``` + +Most of the files are the same: output files, env vars script, test run metadata. The difference for single sbatch mode is that `slurm-job.toml` is created for the entire job as well as `cloudai_sbatch_script.sh`. Extra `common.err`/`common.out` files are created for general sbatch outputs. + + + +### Extra srun and sbatch Arguments + +CloudAI forms sbatch script and srun commands following internal rules. Users can affect the generation by setting special arguments in the system TOML file. + +For example (in a system TOML file): + +``` toml +extra_sbatch_args = [ + "--section=4", + "--other-arg val" +] +``` + +It will result in sbatch file content like this: + +``` bash +... # CloudAI set sbatch arguments +#SBATCH --section=4 +#SBATCH --other-arg val +... +``` + +Another example (in a system TOML file): + +``` toml +extra_srun_args = "--arg=val --other-arg=other-val" +``` + +Will result in srun command inside sbatch script like this: + +``` bash +srun ... --arg=val --other-arg=other-val ... +``` + + + +### Container Mounts + +CloudAI runs all slurm jobs using containers. To simplify file system related tasks, CloudAI mounts the following directories into the container: + +1. Test output directory (`///`, like `results/scenario_2024-06-18_17-40-13/Tests.1/0`) is mounted as `/cloudai_run_results`. + +2. Test specific mounts can be specified in TOML files: + + ``` toml + extra_container_mounts = [ + "/path/to/mount1:/path/in/container1", + "/path/to/mount2:/path/in/container2" + ] + + [cmd_args] + ... + ``` + + These mounts are not verified for validity and do not override default mounts. + +3. Test specific mounts can be mounted in-code. + + + +### Head Node without Shared Storage Available on Compute Nodes + +When compute nodes do not share the file system with head node, `--enable-cache-without-check` for `run` and `dry-run` skips the real check for cache existence, but still builds all paths correctly. The flow is like this: + +1. *\[on the head node\]* run `cloudai install`. +2. *\[on the head node\]* copy cache to compute nodes. +3. Modify `system.toml` to set compute nodes' installation root. +4. Run `cloudai run --enable-cache-without-check ...`. + + + +### Dev Details + +`SlurmCommandGenStrategy` defines abstract method `_container_mounts(tr: TestRun)` that must be implemented by every subclass. This method is used in `SlurmCommandGenStrategy.container_mounts(tr: TestRun)` (defined as `@final`) where mounts like `/cloudai_run_results` (default mount), `TestDefinition.extra_container_mounts` (from Test TOML) and test specific mounts (defined in-code) are added. + + + +### Nsys Tracing + +Users can enable Nsys tracing for any workload when running via Slurm. Note that `nsys` should be available on the compute nodes; CloudAI does not manage it. + +The configuration fields are: + +``` python +enable: bool = True +nsys_binary: str = "nsys" +task: str = "profile" +output: Optional[str] = None +sample: Optional[str] = None +trace: Optional[str] = None +force_overwrite: Optional[bool] = None +capture_range: Optional[str] = None +capture_range_end: Optional[str] = None +cuda_graph_trace: Optional[str] = None +gpu_metrics_devices: Optional[str] = None +extra_args: list[str] = [] +``` + +Fields with `None` value are not passed to `nsys` command. diff --git a/doc/USER_GUIDE.rst b/doc/USER_GUIDE.rst deleted file mode 100644 index 95e65d1f6..000000000 --- a/doc/USER_GUIDE.rst +++ /dev/null @@ -1,613 +0,0 @@ -User Guide -========== - -The purpose of this chapter is to help users utilize and understand the different components of the CloudAI framework. The chapter covers topics such as adding new tests and downloading datasets for running NeMo-launcher. - -System Schema --------------- - -In this section, we introduce the concept of the system schema, explain the meaning of each field, and describe how the fields should be used. The system schema is a TOML file that allows users to define a system's configuration. - -.. code-block:: toml - - name = "example-cluster" - scheduler = "slurm" - - install_path = "./install" - output_path = "./results" - default_partition = "partition_1" - - mpi = "pmix" - gpus_per_node = 8 - ntasks_per_node = 8 - - cache_docker_images_locally = true - - [[partitions]] - name = "partition_1" - - [[partitions.groups]] - name = "group_a" - nodes = ["node-[001-025]"] - - [[partitions.groups]] - name = "group_b" - nodes = ["node-[026-050]"] - - [[partitions]] - name = "partition_2" - - [global_env_vars] - # NCCL Specific Configurations - NCCL_IB_GID_INDEX = "3" - NCCL_IB_TIMEOUT = "20" - NCCL_IB_QPS_PER_CONNECTION = "4" - - # Device Visibility Configuration - MELLANOX_VISIBLE_DEVICES = "0,3,4,5,6,9,10,11" - CUDA_VISIBLE_DEVICES = "0,1,2,3,4,5,6,7" - - -Field Descriptions ------------------- - -.. list-table:: Field Descriptions - :header-rows: 1 - :widths: 25 75 - - * - **Field** - - **Description** - * - **name** - - Specifies the name of the system. Users can choose any name that is convenient for them. - * - **scheduler** - - Indicates the type of system. It should be one of the supported types, currently ``slurm`` or ``standalone``. ``slurm`` refers to a system with the Slurm scheduler, while ``standalone`` refers to a single-node system without any slave nodes. Other values are possible depending on the available schedulers supported by CloudAI. - * - **install_path** - - Specifies the path where test prerequisites are installed. Docker images are downloaded to this path if the user chooses to cache Docker images. Relative paths are converted to absolute paths using the current working directory when the system is loaded, preserving symbolic links. - * - **output_path** - - Defines the default path where outputs are stored. Whenever a user runs a test scenario, a new subdirectory will be created under this path. - * - **default_partition** - - Specifies the default partition where jobs are scheduled. - * - **partitions** - - Describes the available partitions and nodes within those partitions. - * - **[optional] groups** - - Within the same partition, users can define groups of nodes. The group concept can be used to allocate nodes from specific groups in a test scenario schema. For instance, this feature is useful for specifying topology awareness. Groups represent logical partitioning of nodes and users are responsible for ensuring no overlap across groups. - * - **mpi** - - Indicates the Process Management Interface (PMI) implementation to be used for inter-process communication. - * - **gpus_per_node** and **ntasks_per_node** - - These are Slurm arguments passed to the ``sbatch`` script and ``srun``. - * - **cache_docker_images_locally** - - Specifies whether CloudAI should cache remote Docker images locally during installation. If set to ``true``, CloudAI will cache the Docker images, enabling local access without needing to download them each time a test is run. This approach saves network bandwidth but requires more disk capacity. If set to ``false``, CloudAI will allow Slurm to download the Docker images as needed when they are not cached locally by Slurm. - * - **global_env_vars** - - Lists all global environment variables that will be applied globally whenever tests are run. - * - **[optional] slurm_api** - - Uses Slurm REST API instead of local Slurm CLI tools. Set ``url`` and optional ``verify_certs`` and ``headers``. - Header values may reference environment variables using ``${NAME}``. - -Slurm REST API -~~~~~~~~~~~~~~ - -Set ``slurm_api.version`` to match your service: ``v0.0.38`` (the default), ``v0.0.40``, ``v0.0.41``, or ``v0.0.42``. -The service must enable the ``slurm`` endpoint backed by ``slurmctld``. To retrieve status and timing after a job -leaves the controller's records, it must also expose the ``slurmdb`` endpoint backed by ``slurmdbd``. - -Workload steps inherit the system's ``ntasks_per_node`` and ``gpus_per_node`` settings in REST mode, -as they do with ``sbatch``. Explicit ``srun`` options can override these defaults for individual steps. - -.. code-block:: toml - - [slurm_api] - url = "https://slurm-api.example.com" - version = "v0.0.42" - verify_certs = true - - [slurm_api.headers] - X-SLURM-USER-NAME = "${SLURM_USER}" - X-SLURM-USER-TOKEN = "${SLURM_JWT}" - -Workloads that invoke their own Slurm launcher instead of producing an sbatch script are not supported in REST mode. - -RunAI Scheduler ---------------- - -When using RunAI as the scheduler, additional fields must be specified in the system schema TOML file. The following is a list of required fields and how to set them: - -.. code-block:: toml - - name = "runai-cluster" - scheduler = "runai" - - install_path = "./install" - output_path = "./results" - - base_url = "http://runai.example.com" # The URL of your RunAI system, typically the same as used for the web interface. - verify_certs = true # Applies to HTTPS requests and training log connections. - user_email = "your_email" # The email address used to log into the RunAI system. - app_id = "your_app_id" # Obtained by creating an application in the RunAI web interface. - app_secret = "" # Fill in the secret obtained with the app_id. - project_id = "your_project_id" # Project ID assigned or created in the RunAI system (usually an integer). - cluster_id = "your_cluster_id" # Cluster ID in UUID format (e.g., a69928cc-ccaa-48be-bda9-482440f4d855). - -- After logging into the RunAI web interface, navigate to Access → Applications and create a new application to obtain ``app_id`` and ``app_secret`` -- Use your assigned project and cluster IDs. Contact your administrator if they are not available -- All other fields follow the same semantics as in the Slurm system schema (e.g., ``install_path``, ``output_path``) - -RunAI HTTP requests use a 30-second timeout. The optional ``verify_certs`` setting defaults to ``true`` and controls -server TLS certificate verification for both HTTPS requests and training log connections. Set it to ``false`` to disable -verification. For training log connections with a private certificate authority, install its CA certificate in the system trust store. - -Test Scenario Schema --------------------- - -A test scenario is a set of tests with specific dependencies between them. A test scenario is described in a TOML schema file. The following is an example of a test scenario file: - -.. code-block:: toml - - name = "nccl-test" - - [[Tests]] - id = "Tests.1" - test_name = "nccl_test_all_reduce" - num_nodes = "2" - time_limit = "00:20:00" - - [[Tests]] - id = "Tests.2" - test_name = "nccl_test_all_gather" - num_nodes = "2" - time_limit = "00:20:00" - [[Tests.dependencies]] - type = "start_post_comp" - id = "Tests.1" - - [[Tests]] - id = "Tests.3" - test_name = "nccl_test_reduce_scatter" - num_nodes = "2" - time_limit = "00:20:00" - [[Tests.dependencies]] - type = "start_post_comp" - id = "Tests.2" - -The ``name`` field is the test scenario name, which can be any unique identifier for the scenario. Each test has a section name, following the convention ``Tests.1``, ``Tests.2``, etc., with an increasing index. The ``name`` of a test should be specified in this section and must correspond to an entry in the test schema. If a test in a test scenario is not present in the test schema, CloudAI will not be able to identify it. - -There are two ways to specify nodes: - -- Using the ``num_nodes`` field as shown in the example -- Specifying nodes explicitly like ``nodes = ["node-001", "node-002"]`` - -**Note:** When an explicit node list is provided (e.g., ``nodes = ["node-001", "node-002"]``), CloudAI lets Slurm apply the arbitrary distribution policy for task placement. - -Alternatively, you can utilize the groups feature in the system schema to specify nodes like ``nodes = ['PARTITION_NAME:GROUP_NAME:NUM_NODES']``, which allocates ``num_nodes`` from the group name in the specified partition. You can also use ``nodes = ['PARTITION_NAME:GROUP_NAME:max_avail']``, which allocates all the available nodes from the group name in the specified partition. - -You can optionally specify a time limit in the Slurm format. Tests can have dependencies. If no dependencies are specified, all tests will run in parallel. - -CloudAI supports three types of dependencies: - -- ``start_post_init`` -- ``start_post_comp`` -- ``end_post_comp`` - -Dependencies of a test can be described as a subsection of the test. It requires other tests' ``id`` and dependency ``type``. - -- ``start_post_init`` means the test starts after the prior test begins, with a specified delay. -- ``start_post_comp`` means the test starts after the prior test completes. -- ``end_post_comp`` means the test ends when the prior test completes. - -Agents Configuration --------------------- - -For DSE workloads, you can pass agent configuration via ``agent_config`` in the scenario. - -``BaseAgentConfig`` includes: - -- ``random_seed``: controls deterministic random behavior in agents. -- ``start_action``: controls the very first action strategy (``"random"`` or ``"first"``). -- ``rewards`` (optional): nested table mapped to ``RewardOverrides``. When present, ``CloudAIGymEnv`` uses it for - constraint failures and for substituting failed-metric values in observations. - - - ``constraint_failure``: reward returned when ``TestDefinition.constraint_check`` fails on a step. If omitted - or unset, the environment uses ``-1.0``. - - ``metric_failure``: value written into the observation vector when a metric is missing or is the - canonical error sentinel ``METRIC_ERROR`` (a distinct object from numeric metrics, so a valid ``-1.0`` - result is not mistaken for an error). If omitted or unset, observations use ``-1.0`` for that slot. - -Example: - -.. code-block:: toml - - [[Tests]] - id = "Tests.1" - test_name = "nccl_test_all_reduce" - agent = "grid_search" - agent_steps = 10 - agent_metrics = ["default"] - agent_reward_function = "inverse" - - [Tests.agent_config] - random_seed = 123 - start_action = "first" - - [Tests.agent_config.rewards] - constraint_failure = -5.0 - metric_failure = 0.0 - -When an agent honors ``start_action = "first"``, it should start from ``CloudAIGymEnv.first_sweep`` (the sweep -is built from first values of each sweep parameter). ``start_action = "random"`` means starting from a random -action, typically seeded by ``random_seed``. - -Custom agents may extend the ``BaseAgentConfig`` and offer more parameters to configure. - -DSE Parameter Exclusions -~~~~~~~~~~~~~~~~~~~~~~~~ - -CloudAI builds the DSE parameter space implicitly from list-valued fields under ``cmd_args``, list-valued -``extra_env_vars``, and list-valued ``num_nodes``. Most lists mean "try each value", but some workload settings are -real list-valued configuration, such as worker port lists or ordered benchmark phases. - -Use ``dse_excluded_args`` when a list under ``cmd_args`` should stay intact instead of becoming a sweep dimension. -Entries must be dot-separated paths that start with ``cmd_args.`` and may point to either a single field or a parent -object. Matching is prefix-based, so excluding ``cmd_args.foo`` also excludes nested list-valued fields such as -``cmd_args.foo.bar`` from DSE parameter discovery. - -.. code-block:: toml - - [[Tests]] - id = "Tests.1" - test_name = "my_test" - dse_excluded_args = ["cmd_args.lmcache.lmcache_worker_ports"] - - [Tests.cmd_args.lmcache] - chunk_size = [256, 512] - lmcache_worker_ports = [8788, 8789, 8790, 8791] - -In this example, ``cmd_args.lmcache.chunk_size`` is still swept, while -``cmd_args.lmcache.lmcache_worker_ports`` is passed through as one list value. The exclusion does not remove or mutate -the field; it only prevents CloudAI from adding that path to the DSE parameter space. - -``dse_excluded_args`` currently applies only to ``cmd_args`` paths. It does not exclude list-valued ``extra_env_vars`` -or ``num_nodes``; those lists are still interpreted as sweep dimensions. To exclude many nested list fields at once, -exclude their common parent path. Common examples are ``cmd_args.aiperf_phases`` and -``cmd_args.lmcache.lmcache_worker_ports``. - -Metric Errors and Report Strategies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Report generation strategies signal a failed or missing metric by returning the singleton ``METRIC_ERROR`` from -``cloudai.core`` (type ``MetricErrorSentinel``; ``float(METRIC_ERROR)`` is ``-1.0`` for numeric use). The declared -return type of ``ReportGenerationStrategy.get_metric`` is ``MetricValue`` (``float | MetricErrorSentinel``). - -When branching on success vs failure, use **identity** (``value is METRIC_ERROR``), not equality to ``-1.0``, so a legitimate metric of ``-1.0`` is not -treated as an error. The gym observation path uses that identity check before applying ``agent_config.rewards.metric_failure``. - -Configuring HTTP Data Repository --------------------------------- - -The HTTP Data Repository is currently supported for Slurm systems only. To enable access, you must update your system schema file and create a credential file in your CloudAI project's root directory. - -The following steps are required to configure the HTTP Data Repository: - -- :ref:`Step 1: Update the System Schema File ` -- :ref:`Step 2: Create the Credential File ` -- :ref:`Step 3: Usage ` - -.. _step-1-update-system-schema-file: - -Updating the System Schema File -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Add the following section to your system schema TOML file (e.g., ``system_schema.toml``): - -.. code-block:: toml - - [data_repository] - endpoint = "https://my-data-endpoint.com" - -Replace the endpoint with your actual data repository URL. - -.. _step-2-create-credential-file: - -Creating the Credential File -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In the root of your CloudAI project (i.e., the current working directory), create a file named ``.cloudai.toml`` with the following content: - -.. code-block:: toml - - [data_repository] - token = "" - -Replace ```` with your actual token. - -.. _step-3-usage: - -Usage -~~~~~~~~~~~~~ - -Both the endpoint and token must be valid for the HTTP Data Repository to function correctly. If either is missing or incorrect, data will not be posted. - -Using Test Hooks in CloudAI ---------------------------- - -A test hook in CloudAI is a specialized test that runs either before or after each main test in a scenario, providing flexibility to prepare the environment or clean up resources. Hooks are defined as pre-test or post-test and referenced in the test scenario’s TOML file using ``pre_test`` and ``post_test`` fields. - -.. code-block:: toml - - name = "nccl-test" - - pre_test = "nccl_test_pre" - post_test = "nccl_test_post" - - [[Tests]] - id = "Tests.1" - test_name = "nccl_test_all_reduce" - time_limit = "00:20:00" - -CloudAI organizes hooks in a dedicated directory structure: - -- Hook directory: All hook configurations reside in ``conf/hook/`` -- Hook test scenarios: Place pre-test hook and post-test hook scenario files in ``conf/hook/`` -- Hook tests: Place individual tests referenced in hooks within ``conf/hook/test/`` - -In the execution flow, pre-test hooks run before the main test, which only executes if the pre-test completes successfully. Post-test hooks follow the main test, provided the prior steps succeed. If a pre-test hook fails, the main test and its post-test hook are skipped. - -.. code-block:: toml - - name = "nccl_test_pre" - - [[Tests]] - id = "Tests.1" - test_name = "nccl_test_all_reduce" - time_limit = "00:20:00" - -Downloading DeepSeek Weights ----------------------------- - -To run DeepSeek R1 tests in CloudAI, you must download the model weights in advance. These weights are distributed via the NVIDIA NGC Registry and must be manually downloaded using the NGC CLI. - -The following steps are required to download the DeepSeek weights: - -- :ref:`Step 1: Installing NGC CLI ` -- :ref:`Step 2: Configuring NGC CLI ` -- :ref:`Step 3: Downloading the Weights ` -- :ref:`Step 4: Verifying the Download ` - -.. _step-1-installing-ngc-cli: - -Installing NGC CLI -~~~~~~~~~~~~~~~~~~~ - -Download and install the NGC CLI using the following commands: - -.. code-block:: bash - - wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/3.64.2/files/ngccli_linux.zip -O ngccli_linux.zip - unzip ngccli_linux.zip - chmod u+x ngc-cli/ngc - echo "export PATH=\"$PATH:$(pwd)/ngc-cli\"" >> ~/.bash_profile && source ~/.bash_profile - -This will make the ``ngc`` command available in your terminal. - -.. _step-2-configuring-ngc-cli: - -Configuring NGC CLI -~~~~~~~~~~~~~~~~~~~ - -Authenticate your CLI with your NGC API key by running: - -.. code-block:: bash - - ngc config set - -When prompted, paste your API key, which you can obtain from https://org.ngc.nvidia.com/setup. - -.. _step-3-downloading-the-weights: - -Downloading the Weights -~~~~~~~~~~~~~~~~~~~~~~~~ - -Navigate to the directory where you want the DeepSeek model weights to be stored, then run: - -.. code-block:: bash - - ngc registry model download-version nim/deepseek-ai/deepseek-r1-instruct:hf-5dde110-nim-fp8 --dest . - -This command will create a folder named: - -.. code-block:: text - - deepseek-r1-instruct_vhf-5dde110-nim-fp8/ - -inside your current directory. - -.. _step-4-verifying-the-download: - -Verifying the Download -~~~~~~~~~~~~~~~~~~~~~~~ - -Make sure the full model has been downloaded by checking the folder size: - -.. code-block:: bash - - du -sh deepseek-r1-instruct_vhf-5dde110-nim-fp8 - -The expected size is approximately 642 GB. If it’s significantly smaller, remove the folder and re-run the download. - -Slurm ------ - -The following are subtopics related to Slurm: - -Single sbatch vs Per-case sbatch -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -CloudAI supports two modes for submitting Slurm jobs: - -1. (default) Per-case sbatch mode: each case is submitted as a separate sbatch job, allowing flexible scheduling and dependency management. - - - Suits best when cases can run in parallel or there are no dependencies between cases. - -2. Single sbatch mode: all cases are submitted together in a single sbatch job and share the same nodes. Each next case starts after the previous one completes. To enable it one needs to pass ``--single-sbatch`` flag to ``cloudai run`` command (works only for Slurm systems). - - - Suits best for jobs when cases need to run on the same nodes for performance reasons. - - There is no support for dependency management between cases yet; all jobs run one after another. - -Assuming two cases in a scenario like this: - -.. code-block:: toml - - [[Tests]] - id = "nccl.all_reduce" - test_name = "nccl-all_reduce" - num_nodes = 2 - time_limit = "00:20:00" - - [[Tests]] - id = "nccl.all_gather" - test_name = "nccl-all_gather" - num_nodes = 2 - time_limit = "00:20:00" - -Regular output directory structure (some files are omitted for clarity): - -.. code-block:: bash - - $ tree results/scenario - results/scenario - ├── nccl.all_gather - │ └── 0 - │ ├── cloudai_sbatch_script.sh - │ ├── env_vars.sh - │ ├── metadata/ - │ ├── stderr.txt - │ ├── stdout.txt - │ └── test-run.toml - └── nccl.all_reduce - └── 0 - ├── cloudai_sbatch_script.sh - ├── env_vars.sh - ├── metadata/ - ├── stderr.txt - ├── stdout.txt - └── test-run.toml - -Single sbatch mode output directory structure: - -.. code-block:: bash - - $ tree results/scenario - results/scenario - ├── cloudai_sbatch_script.sh - ├── common.err - ├── common.out - ├── metadata/ - ├── nccl.all_gather - │ └── 0 - │ ├── env_vars.sh - │ ├── stderr.txt - │ ├── stdout.txt - │ └── test-run.toml - ├── nccl.all_reduce - │ └── 0 - │ ├── env_vars.sh - │ ├── stderr.txt - │ ├── stdout.txt - │ └── test-run.toml - └── slurm-job.toml - -Most of the files are the same: output files, env vars script, test run metadata. The difference for single sbatch mode is that ``slurm-job.toml`` is created for the entire job as well as ``cloudai_sbatch_script.sh``. Extra ``common.err``/``common.out`` files are created for general sbatch outputs. - -Extra srun and sbatch Arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -CloudAI forms sbatch script and srun commands following internal rules. Users can affect the generation by setting special arguments in the system TOML file. - -For example (in a system TOML file): - -.. code-block:: toml - - extra_sbatch_args = [ - "--section=4", - "--other-arg val" - ] - -It will result in sbatch file content like this: - -.. code-block:: bash - - ... # CloudAI set sbatch arguments - #SBATCH --section=4 - #SBATCH --other-arg val - ... - -Another example (in a system TOML file): - -.. code-block:: toml - - extra_srun_args = "--arg=val --other-arg=other-val" - -Will result in srun command inside sbatch script like this: - -.. code-block:: bash - - srun ... --arg=val --other-arg=other-val ... - -Container Mounts -~~~~~~~~~~~~~~~~ - -CloudAI runs all slurm jobs using containers. To simplify file system related tasks, CloudAI mounts the following directories into the container: - -1. Test output directory (``///``, like ``results/scenario_2024-06-18_17-40-13/Tests.1/0``) is mounted as ``/cloudai_run_results``. -2. Test specific mounts can be specified in TOML files: - - .. code-block:: toml - - extra_container_mounts = [ - "/path/to/mount1:/path/in/container1", - "/path/to/mount2:/path/in/container2" - ] - - [cmd_args] - ... - - These mounts are not verified for validity and do not override default mounts. - -3. Test specific mounts can be mounted in-code. - -Head Node without Shared Storage Available on Compute Nodes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When compute nodes do not share the file system with head node, ``--enable-cache-without-check`` for ``run`` and ``dry-run`` skips the real check for cache existence, but still builds all paths correctly. The flow is like this: - -1. *[on the head node]* run ``cloudai install``. -2. *[on the head node]* copy cache to compute nodes. -3. Modify ``system.toml`` to set compute nodes' installation root. -4. Run ``cloudai run --enable-cache-without-check ...``. - -Dev Details -~~~~~~~~~~~ - -``SlurmCommandGenStrategy`` defines abstract method ``_container_mounts(tr: TestRun)`` that must be implemented by every subclass. This method is used in ``SlurmCommandGenStrategy.container_mounts(tr: TestRun)`` (defined as ``@final``) where mounts like ``/cloudai_run_results`` (default mount), ``TestDefinition.extra_container_mounts`` (from Test TOML) and test specific mounts (defined in-code) are added. - -Nsys Tracing -~~~~~~~~~~~~ - -Users can enable Nsys tracing for any workload when running via Slurm. Note that ``nsys`` should be available on the compute nodes; CloudAI does not manage it. - -The configuration fields are: - -.. code-block:: python - - enable: bool = True - nsys_binary: str = "nsys" - task: str = "profile" - output: Optional[str] = None - sample: Optional[str] = None - trace: Optional[str] = None - force_overwrite: Optional[bool] = None - capture_range: Optional[str] = None - capture_range_end: Optional[str] = None - cuda_graph_trace: Optional[str] = None - gpu_metrics_devices: Optional[str] = None - extra_args: list[str] = [] - -Fields with ``None`` value are not passed to ``nsys`` command. diff --git a/doc/api.mdx b/doc/api.mdx new file mode 100644 index 000000000..3f1128603 --- /dev/null +++ b/doc/api.mdx @@ -0,0 +1,28 @@ + + +`cloudai.api` lets a Python program validate or run a scenario and find its saved results. Pass configuration file paths as `pathlib.Path` objects: + +``` python +from pathlib import Path + +import cloudai.api + +system = Path("conf/common/system/example_slurm_cluster.toml") +scenario = Path("conf/common/test_scenario/sleep.toml") + +cloudai.api.validate_scenario( + scenario, system, tests_dir=Path("conf/common/test") +) +experiment = cloudai.api.run_experiment( + scenario, system, tests_dir=Path("conf/common/test") +) +print(experiment.status, experiment.path) +``` + +`validate_scenario` performs the same configuration loading checks as `run_experiment`. It returns `None` on success and raises configuration errors on failure, without creating a runner, installing dependencies, or submitting jobs. Pass `tests_dir` for referenced test definitions and `hook_dir` for custom hooks, as with `run_experiment`. Validation does not check cluster availability or whether a workload will run successfully. + +`run_experiment` waits for the scenario to finish and returns an `Experiment` model with the same data saved in `experiment.json`. Workload failures appear in its status. Configuration and setup errors raise exceptions. Use `mode="dry-run"` to generate commands without running workloads. Use `single_sbatch=True` for a single Slurm allocation. Pass `hook_dir` to use a custom hook directory. Pass an `on_start` callback to receive the initial `Experiment` and a cancellation function after `experiment.json` is created and before execution begins. The function remains synchronous after calling the callback. + +`list_experiments(system)` returns `(experiment_id, result_directory)` pairs from the system's configured output directory. Directories without `experiment.json` are skipped. A missing output directory gives an empty list; experiment file contents are not parsed. + +`get_experiment(experiment_id, system)` loads an `Experiment` from that system's output directory. Pass a `Path` to a result directory to load it directly. Invalid IDs and missing, unreadable, or malformed experiment files raise `ValueError`. diff --git a/doc/api.rst b/doc/api.rst deleted file mode 100644 index 8f172ae51..000000000 --- a/doc/api.rst +++ /dev/null @@ -1,49 +0,0 @@ -Python API -========== - -``cloudai.api`` lets a Python program validate or run a scenario and find its saved results. -Pass configuration file paths as ``pathlib.Path`` objects: - -.. code-block:: python - - from pathlib import Path - - import cloudai.api - - system = Path("conf/common/system/example_slurm_cluster.toml") - scenario = Path("conf/common/test_scenario/sleep.toml") - - cloudai.api.validate_scenario( - scenario, system, tests_dir=Path("conf/common/test") - ) - experiment = cloudai.api.run_experiment( - scenario, system, tests_dir=Path("conf/common/test") - ) - print(experiment.status, experiment.path) - -``validate_scenario`` performs the same configuration loading checks as -``run_experiment``. It returns ``None`` on success and raises configuration errors -on failure, without creating a runner, installing dependencies, or submitting -jobs. Pass ``tests_dir`` for referenced test definitions and ``hook_dir`` for -custom hooks, as with ``run_experiment``. Validation does not check cluster -availability or whether a workload will run successfully. - -``run_experiment`` waits for the scenario to finish and returns an ``Experiment`` -model with the same data saved in ``experiment.json``. Workload failures appear in -its status. Configuration and setup errors raise exceptions. Use -``mode="dry-run"`` to generate commands without running workloads. Use -``single_sbatch=True`` for a single Slurm allocation. Pass ``hook_dir`` to use a -custom hook directory. Pass an ``on_start`` callback to receive the initial -``Experiment`` and a cancellation function after ``experiment.json`` is created -and before execution begins. The function remains synchronous after calling the -callback. - -``list_experiments(system)`` returns ``(experiment_id, result_directory)`` pairs -from the system's configured output directory. Directories without -``experiment.json`` are skipped. A missing output directory gives an empty list; -experiment file contents are not parsed. - -``get_experiment(experiment_id, system)`` loads an ``Experiment`` from that -system's output directory. Pass a ``Path`` to a result directory to load it -directly. Invalid IDs and missing, unreadable, or malformed experiment files -raise ``ValueError``. diff --git a/doc/conf.py b/doc/conf.py deleted file mode 100644 index 96fed0e0b..000000000 --- a/doc/conf.py +++ /dev/null @@ -1,114 +0,0 @@ -# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -import inspect -import os -import re -import sys -from pathlib import Path - -# Add the project source to Python path for autodoc -sys.path.insert(0, os.path.abspath("../src")) - - -# Custom autodoc processing to clean up Pydantic classes -def autodoc_skip_member(app, what, name, obj, skip, options): - """Skip unwanted Pydantic and other internal members.""" - exclude_patterns = {re.compile(r"model_.*")} - - if any(pattern.match(name) for pattern in exclude_patterns): - return True - - # Skip private methods starting with underscore (except __init__) - if name.startswith("_") and name != "__init__": - return True - - # On the systems page, skip all methods and properties from pydantic models — - # only TOML-configurable fields are relevant. Pydantic fields bypass this hook entirely. - non_attr = ( - inspect.isfunction(obj) or inspect.ismethod(obj) or isinstance(obj, (classmethod, staticmethod, property)) - ) - if what == "pydantic_model" and app.env.docname == "systems" and non_attr: - return True - - return skip - - -def setup(app): - app.connect("autodoc-skip-member", autodoc_skip_member) - - -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = "CloudAI" -copyright = "2026, NVIDIA CORPORATION & AFFILIATES" -author = "NVIDIA CORPORATION & AFFILIATES" -version = "1.6.0-beta" -release = "1.6.0-beta" - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - - -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.viewcode", - "sphinx.ext.napoleon", - "sphinx.ext.autosummary", - "sphinxcontrib.mermaid", - "sphinx_copybutton", - "sphinxcontrib.autodoc_pydantic", -] - -exclude_patterns = ["_build"] - -# -- Autodoc configuration --------------------------------------------------- -autodoc_default_options = { - "members": True, - "member-order": "bysource", - "special-members": "__init__", - "inherited-members": "BaseModel", -} - -autodoc_pydantic_model_show_json = False -autodoc_pydantic_model_show_field_summary = False -autodoc_pydantic_model_show_config_summary = False -autodoc_pydantic_field_list_validators = False -autodoc_pydantic_model_show_validator_summary = False -autodoc_pydantic_model_show_validator_members = False - -# Generate autosummary even if no references -autosummary_generate = True - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -is_cms_build = os.environ.get("CLOUDAI_DOC_TARGET") == "cms" -html_theme, mermaid_output_format, mermaid_cmd, mermaid_params = "nvidia_sphinx_theme", "raw", "", [] -if is_cms_build: - html_theme = "sphinx_rtd_theme" - mermaid_output_format = "png" - mermaid_cmd = str(Path(__file__).parent / "_build-cms/node_modules/.bin/mmdc") - mermaid_params = ["-p", str(Path(__file__).parent / "_build-cms/puppeteer-config.json")] - -# Set the root document to index -root_doc = "index" - -source_suffix = [".rst"] diff --git a/doc/generate_reference.py b/doc/generate_reference.py new file mode 100644 index 000000000..de80f19dd --- /dev/null +++ b/doc/generate_reference.py @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate Fern snippets from Python objects; authored MDX is never rewritten.""" + +import dataclasses +import html +import importlib +import inspect +import sys +from pathlib import Path + +import griffe +import jinja2 +import pydantic +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def mdx_text(value): + """Escape prose leaves for MDX without interpreting or rewriting Markdown.""" + return html.escape(str(value), quote=False).translate(str.maketrans({"{": "{", "}": "}"})) + + +def load_class(symbol): + module, name = symbol.rsplit(".", 1) + return getattr(importlib.import_module(module), name) + + +def describe_fields(cls, obj, exclude): + """Read fields without evaluating their default factories.""" + fields = [] + if issubclass(cls, pydantic.BaseModel): + for name, field in cls.model_fields.items(): + if name in exclude: + continue + source = obj.all_members.get(name) + description = field.description or (source.docstring.value if source and source.docstring else "") + factory = field.default_factory + fields.append( + { + "name": name, + "annotation": inspect.formatannotation(field.annotation), + "description": description, + "required": field.is_required(), + "default": repr(field.default) if not field.is_required() and factory is None else None, + "factory": getattr(factory, "__qualname__", str(factory)) if factory else None, + "constraints": [repr(item) for item in field.metadata], + "alias": field.alias, + } + ) + elif dataclasses.is_dataclass(cls): + for field in dataclasses.fields(cls): + if field.name in exclude: + continue + source = obj.all_members.get(field.name) + fields.append( + { + "name": field.name, + "annotation": inspect.formatannotation(field.type), + "description": source.docstring.value if source and source.docstring else "", + "required": field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING, + "default": repr(field.default) if field.default is not dataclasses.MISSING else None, + "factory": getattr(field.default_factory, "__qualname__", str(field.default_factory)) + if field.default_factory is not dataclasses.MISSING + else None, + "constraints": [], + "alias": None, + } + ) + return fields + + +def describe_members(cls, obj, exclude): + """Read documented public members, excluding model implementation details.""" + members = [] + for name, member in obj.all_members.items(): + if member.canonical_path.startswith("pydantic."): + continue + if name in exclude or (name.startswith("_") and name != "__init__") or name.startswith("model_"): + continue + if not member.docstring: + continue + runtime = inspect.getattr_static(cls, name, None) + # Pydantic validators were hidden by the previous documentation configuration. + decorators = getattr(cls, "__pydantic_decorators__", None) + if decorators and (name in decorators.field_validators or name in decorators.model_validators): + continue + if isinstance(runtime, (staticmethod, classmethod)): + runtime = runtime.__func__ + if isinstance(runtime, property): + signature = ": " + str(getattr(member, "annotation", "")) + elif callable(runtime): + signature = str(inspect.signature(runtime)) + else: + signature = ": " + str(getattr(member, "annotation", "")) + members.append({"name": name, "signature": signature, "sections": member.docstring.parsed}) + return members + + +def describe_class(symbol, obj, exclude=(), fields_only=False): + """Combine Griffe's docstrings with authoritative runtime field metadata.""" + cls = load_class(symbol) + fields = describe_fields(cls, obj, exclude) + excluded = set(exclude) | {field["name"] for field in fields} + members = [] if fields_only else describe_members(cls, obj, excluded) + return { + "symbol": symbol, + "description": obj.docstring.value if obj.docstring else "", + "fields": fields, + "members": members, + } + + +def generate(root=ROOT): + config = yaml.safe_load((root / "fern/reference.yml").read_text()) + loader = griffe.GriffeLoader( + search_paths=[str(root / "src"), *sys.path], + extensions=griffe.load_extensions("griffe_pydantic"), + docstring_parser=griffe.Parser.google, + ) + loader.load("cloudai") + loader.resolve_aliases(external=True) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(root / "doc/templates"), + undefined=jinja2.StrictUndefined, + keep_trailing_newline=True, + autoescape=False, + ) + env.filters["mdx"] = mdx_text + template = env.get_template("reference.mdx.jinja") + output = root / "fern/generated" + output.mkdir(parents=True, exist_ok=True) + expected = set() + for entry in config["references"]: + symbol = entry["symbol"] + data = describe_class( + symbol, loader.modules_collection[symbol], entry.get("exclude", []), entry.get("fields_only", False) + ) + path = output / f"{symbol}.mdx" + path.write_text(template.render(**data)) + expected.add(path) + for stale in set(output.glob("*.mdx")) - expected: + stale.unlink() + print(f"Generated {len(expected)} reference snippets in {output}") + + +if __name__ == "__main__": + generate() diff --git a/doc/index.mdx b/doc/index.mdx new file mode 100644 index 000000000..d420815d5 --- /dev/null +++ b/doc/index.mdx @@ -0,0 +1,210 @@ + + + + +## Overview + +CloudAI benchmark framework aims to develop an industry standard benchmark focused on grading Data Center (DC) scale AI systems in the cloud. The primary motivation is to provide automated benchmarking on various systems. + +This document contains the following chapters: + +- [Tutorial](Tutorial.mdx) +- [User Guide](USER_GUIDE.mdx) +- [Python API](api.mdx) +- [Reporting](reporting.mdx) +- [Systems](systems.mdx) +- [Workloads](workloads/index.mdx) +- [Development](DEV.mdx) +- [Troubleshooting](troubleshooting.mdx) + + + +## Getting Started + +CloudAI officially targets Python 3.14 while remaining compatible with Python 3.10 and newer. + +Use uv 0.13.4 or newer to install and run CloudAI. For pip-based installation, use pip 26.2.2 or newer. These versions include certificate trust updates for the eMudhra/emSign CA compromise. + +``` bash +git clone git@github.com:NVIDIA/cloudai.git +cd cloudai +uv run cloudai --help +``` + + + +For more instructions on how to set up access for `enroot`, see [Installation Requirements](workloads/workloads_requirements_installation.mdx). + + + + + +### pip-based Installation + +See the preferred Python version in the `.python-version` file and make sure you have it installed (For installation, see [Custom Python Version Installation](index.mdx#install-custom-python-version)). Python 3.10 and newer remain supported. Follow these steps: + +``` bash +git clone git@github.com:NVIDIA/cloudai.git +cd cloudai +python3.14 -m venv venv +source venv/bin/activate +pip install -e . +``` + + + +When upgrading an existing installation, also update the host's `ca-certificates` package and the certificate bundles and installers in existing workload environments and container images. Updating CloudAI does not update these environments. + + + + + +### Custom Python Version Installation + +If your system Python version is not supported, you can install a custom version using the [uv](https://docs.astral.sh/uv/getting-started/installation/) tool: + +``` bash +curl -LsSf https://astral.sh/uv/install.sh | sh +source $HOME/.local/bin/env +uv venv --seed # picks the python version from .python-version + # --seed installs pip and setuptools +source .venv/bin/activate +``` + + + +## Key Concepts + +CloudAI operates on three main schemas: + +- **System Schema**: Describes the system, including the scheduler type, node list, and global environment variables +- **Test Schema**: An instance of a test template with custom arguments and environment variables +- **Test Scenario Schema**: A set of tests with dependencies and additional descriptions about the test scenario + +These schemas enable CloudAI to be flexible and compatible with different systems and configurations. + + + +## CloudAI Modes Usage Examples + +The following are the global options for the `cloudai` command: + +- `--log-file `: specify a file to log output; by default `debug.log` in the current directory is used. Contains log entries of level `DEBUG` and higher +- `--log-level `: specify logging level for standard output; default is `INFO` + + + +### Available Modes + +- [run](index.mdx#run) +- [dry-run](index.mdx#dry-run) +- [generate-report](index.mdx#generate-report) +- [install](index.mdx#install) +- [uninstall](index.mdx#uninstall) +- [list](index.mdx#list) +- [verify-configs](index.mdx#verify-configs) + + + +### Usage Examples + + + +#### run + +This mode runs workloads. It automatically installs prerequisites if they are not met. + +``` bash +cloudai run\ + --system-config conf/common/system/example_slurm_cluster.toml\ + --tests-dir conf/common/test\ + --test-scenario conf/common/test_scenario/sleep.toml +``` + + + +#### dry-run + +This mode simulates running experiments without actually executing them. This is useful for verifying configurations and testing experiment setups. + +``` bash +cloudai dry-run\ + --system-config conf/common/system/example_slurm_cluster.toml\ + --tests-dir conf/common/test\ + --test-scenario conf/common/test_scenario/sleep.toml +``` + + + +#### generate-report + +This mode generates reports under the scenario directory. It automatically runs as part of the `run` mode after experiments are completed. + +``` bash +cloudai generate-report\ + --system-config conf/common/system/example_slurm_cluster.toml\ + --tests-dir conf/common/test\ + --test-scenario conf/common/test_scenario/sleep.toml\ + --result-dir /path/to/result_directory +``` + + + +#### install + +This mode installs test prerequisites. For more details, refer to the [Installation Requirements](workloads/workloads_requirements_installation.mdx) guide. It automatically runs as part of the `run` mode if prerequisites are not met. + +``` bash +cloudai install\ + --system-config conf/common/system/example_slurm_cluster.toml\ + --tests-dir conf/common/test\ + --test-scenario conf/common/test_scenario/sleep.toml +``` + + + +#### uninstall + +This mode is the opposite of the install mode. This mode removes installed test prerequisites. + +``` bash +cloudai uninstall\ + --system-config conf/common/system/example_slurm_cluster.toml\ + --tests-dir conf/common/test\ + --test-scenario conf/common/test_scenario/sleep.toml +``` + + + +#### list + +This mode lists internal components available within CloudAI. + +``` bash +cloudai list +``` + + + +#### verify-configs + +This mode verifies the correctness of system, test, and test scenario configuration files. + +``` bash +# verify all at once +cloudai verify-configs conf + +# verify a single file +cloudai verify-configs conf/common/system/example_slurm_cluster.toml + +# verify all scenarios using specific folder with Test TOMLs +cloudai verify-configs --tests-dir conf/release/spcx/l40s/test conf/release/spcx/l40s/test_scenario +``` + + + +## Documentation Update History + +| Version | Date | Description | +|---------|----------------|------------------------------------------------| +| v1.5.0 | April xx, 2026 | Initial release of this documentation version. | diff --git a/doc/index.rst b/doc/index.rst deleted file mode 100644 index 2d4971361..000000000 --- a/doc/index.rst +++ /dev/null @@ -1,227 +0,0 @@ -CloudAI Benchmark Framework -=========================== - -Overview --------- - -CloudAI benchmark framework aims to develop an industry standard benchmark focused on grading Data Center (DC) scale AI systems in the cloud. The primary motivation is to provide automated benchmarking on various systems. - -This document contains the following chapters: - -.. toctree:: - :hidden: - - Tutorial - USER_GUIDE - api - reporting - systems - workloads/index - DEV - troubleshooting - -- :doc:`Tutorial` -- :doc:`USER_GUIDE` -- :doc:`api` -- :doc:`reporting` -- :doc:`systems` -- :doc:`workloads/index` -- :doc:`DEV` -- :doc:`troubleshooting` - -Getting Started ---------------- - -CloudAI officially targets Python 3.14 while remaining compatible with Python 3.10 and newer. - -Use uv 0.13.4 or newer to install and run CloudAI. For pip-based installation, use pip 26.2.2 or newer. -These versions include certificate trust updates for the eMudhra/emSign CA compromise. - -.. code-block:: bash - - git clone git@github.com:NVIDIA/cloudai.git - cd cloudai - uv run cloudai --help - -.. note:: - - For more instructions on how to set up access for ``enroot``, see :doc:`workloads/workloads_requirements_installation`. - -pip-based Installation -~~~~~~~~~~~~~~~~~~~~~~~ - -See the preferred Python version in the ``.python-version`` file and make sure you have it installed -(For installation, see :ref:`install-custom-python-version`). Python 3.10 and newer remain supported. -Follow these steps: - -.. code-block:: bash - - git clone git@github.com:NVIDIA/cloudai.git - cd cloudai - python3.14 -m venv venv - source venv/bin/activate - pip install -e . - -.. note:: - - When upgrading an existing installation, also update the host's ``ca-certificates`` package and the - certificate bundles and installers in existing workload environments and container images. - Updating CloudAI does not update these environments. - - -.. _install-custom-python-version: - -Custom Python Version Installation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If your system Python version is not supported, you can install a custom version using the `uv `_ tool: - -.. code-block:: bash - - curl -LsSf https://astral.sh/uv/install.sh | sh - source $HOME/.local/bin/env - uv venv --seed # picks the python version from .python-version - # --seed installs pip and setuptools - source .venv/bin/activate - -Key Concepts ------------- - -CloudAI operates on three main schemas: - -- **System Schema**: Describes the system, including the scheduler type, node list, and global environment variables -- **Test Schema**: An instance of a test template with custom arguments and environment variables -- **Test Scenario Schema**: A set of tests with dependencies and additional descriptions about the test scenario - -These schemas enable CloudAI to be flexible and compatible with different systems and configurations. - -CloudAI Modes Usage Examples ------------------------------ - -The following are the global options for the ``cloudai`` command: - -- ``--log-file ``: specify a file to log output; by default ``debug.log`` in the current directory is used. Contains log entries of level ``DEBUG`` and higher -- ``--log-level ``: specify logging level for standard output; default is ``INFO`` - -Available Modes -~~~~~~~~~~~~~~~~ - -- :ref:`run` -- :ref:`dry-run` -- :ref:`generate-report` -- :ref:`install` -- :ref:`uninstall` -- :ref:`list` -- :ref:`verify-configs` - -Usage Examples -~~~~~~~~~~~~~~~ - -.. _run: - -run -^^^ - -This mode runs workloads. It automatically installs prerequisites if they are not met. - -.. code-block:: bash - - cloudai run\ - --system-config conf/common/system/example_slurm_cluster.toml\ - --tests-dir conf/common/test\ - --test-scenario conf/common/test_scenario/sleep.toml - -.. _dry-run: - -dry-run -^^^^^^^ - -This mode simulates running experiments without actually executing them. This is useful for verifying configurations and testing experiment setups. - -.. code-block:: bash - - cloudai dry-run\ - --system-config conf/common/system/example_slurm_cluster.toml\ - --tests-dir conf/common/test\ - --test-scenario conf/common/test_scenario/sleep.toml - -.. _generate-report: - -generate-report -^^^^^^^^^^^^^^^ - -This mode generates reports under the scenario directory. It automatically runs as part of the ``run`` mode after experiments are completed. - -.. code-block:: bash - - cloudai generate-report\ - --system-config conf/common/system/example_slurm_cluster.toml\ - --tests-dir conf/common/test\ - --test-scenario conf/common/test_scenario/sleep.toml\ - --result-dir /path/to/result_directory - -.. _install: - -install -^^^^^^^ - -This mode installs test prerequisites. For more details, refer to the :doc:`workloads/workloads_requirements_installation` guide. It automatically runs as part of the ``run`` mode if prerequisites are not met. - -.. code-block:: bash - - cloudai install\ - --system-config conf/common/system/example_slurm_cluster.toml\ - --tests-dir conf/common/test\ - --test-scenario conf/common/test_scenario/sleep.toml - -.. _uninstall: - -uninstall -^^^^^^^^^ - -This mode is the opposite of the install mode. This mode removes installed test prerequisites. - -.. code-block:: bash - - cloudai uninstall\ - --system-config conf/common/system/example_slurm_cluster.toml\ - --tests-dir conf/common/test\ - --test-scenario conf/common/test_scenario/sleep.toml - -.. _list: - -list -^^^^ - -This mode lists internal components available within CloudAI. - -.. code-block:: bash - - cloudai list - -.. _verify-configs: - -verify-configs -^^^^^^^^^^^^^^ - -This mode verifies the correctness of system, test, and test scenario configuration files. - -.. code-block:: bash - - # verify all at once - cloudai verify-configs conf - - # verify a single file - cloudai verify-configs conf/common/system/example_slurm_cluster.toml - - # verify all scenarios using specific folder with Test TOMLs - cloudai verify-configs --tests-dir conf/release/spcx/l40s/test conf/release/spcx/l40s/test_scenario - -Documentation Update History ------------------------------ - -+----------+-----------------+--------------------------------------------------+ -| Version | Date | Description | -+==========+=================+==================================================+ -| v1.5.0 | April xx, 2026 | Initial release of this documentation version. | -+----------+-----------------+--------------------------------------------------+ diff --git a/doc/pareto-frontiers.mdx b/doc/pareto-frontiers.mdx new file mode 100644 index 000000000..a93e05f06 --- /dev/null +++ b/doc/pareto-frontiers.mdx @@ -0,0 +1,111 @@ + + +The `pareto` scenario report writes `pareto.html` in the scenario results directory. Each plot compares two metrics. Successful iterations and sweep steps supply the points: one test case can contribute several points, and several cases can contribute to one frontier. + + + +## Usage Example + +For [vLLM](workloads/vllm.mdx) or [SGLang](workloads/sglang.mdx), add this block to your scenario TOML: + +``` toml +[reports.pareto] +enable = true + +[[reports.pareto.plots]] +name = "Streaming latency vs throughput" +x = { metric = "tpot", match = { statistic = "p99" } } +y = { metric = "output_token_throughput" } +frontiers = [{ name = "All configurations" }] +``` + +This selects P99 time per output token on X and total output tokens/s on Y. CloudAI minimizes latency and maximizes throughput using the metrics' default directions. Both axes increase normally. + +`frontiers` defines the comparison groups in the plot. Omitting `cases` or setting `cases = []` includes all cases in the scenario. Use this example when all cases belong to the same comparison. + + + +## Available Metrics + +Use the metric keys below in either `x.metric` or `y.metric`. Each axis must select exactly one measurement from each execution. `match` selects among measurements of the same metric, such as different statistics or message sizes. Missing or ambiguous measurements are listed as exclusions in the report. + +Both `vllm` and `sglang` expose these metrics: + +| Metric | Unit | Default direction | +|---------------------------------------------|--------------|-------------------| +| `ttft` | ms | minimize | +| `tpot` | ms | minimize | +| `output_token_throughput` | tokens/s | maximize | +| `request_throughput` | requests/s | maximize | +| `output_token_throughput_per_allocated_gpu` | tokens/s/GPU | maximize | +| `output_token_throughput_per_serving_gpu` | tokens/s/GPU | maximize | +| `accuracy` | ratio | maximize | + +`ttft` measures the wait for the first token; `tpot` measures time per output token after the first token. For either metric, select `mean`, `median`, or `p99` with `match = { statistic = "p99" }`. The throughput metrics have no statistic selector. `accuracy` is available when semantic validation is configured and its result is present; see the workload's Semantic Validation section. + +The two GPU-normalized metrics divide total output tokens/s by different counts: + +- `output_token_throughput_per_allocated_gpu` uses the execution's `num_nodes` times the system's `gpus_per_node`. It includes idle GPUs on those nodes. A larger single-sbatch reservation does not change the individual execution's count. +- `output_token_throughput_per_serving_gpu` uses the configured serving layout, summing prefill and decode GPUs when both roles are configured. It describes configured devices, not measured GPU utilization. + +Choose the denominator that matches your comparison and use it consistently. A normalized metric is omitted when its GPU count cannot be determined. + + + +## Examples and Best Practices + +For inference, start with one of these comparisons: + +- **First-token wait vs capacity:** `ttft` on X, `output_token_throughput` on Y. +- **Streaming latency vs serving efficiency:** `tpot` on X, `output_token_throughput_per_serving_gpu` on Y. +- **Streaming latency vs allocated GPU efficiency:** `tpot` on X, `output_token_throughput_per_allocated_gpu` on Y. + +Keep the model, input/output token lengths, precision, and traffic pattern comparable. Vary concurrency or serving parameters to measure the tradeoff. Use P99 for tail latency, or select mean or median when that is the statistic your comparison requires. Match the statistic and GPU denominator when comparing with another benchmark report. + + + +### Multiple Plots + +Append another `[[reports.pareto.plots]]` block to generate a separate chart. For example, add a first-token latency plot alongside the streaming-latency example above: + +``` toml +[[reports.pareto.plots]] +name = "First-token latency vs throughput" +x = { metric = "ttft", match = { statistic = "p99" } } +y = { metric = "output_token_throughput" } +frontiers = [{ name = "All configurations" }] +``` + + + +### Comparing Versions or Configurations + +Give each version its own frontier so its measurements compete independently. Append this plot to an enabled inference report, replacing the `cases` entries with your scenario's `[[Tests]].id` values: + +``` toml +[[reports.pareto.plots]] +name = "Streaming efficiency by version" +x = { metric = "tpot", match = { statistic = "p99" } } +y = { metric = "output_token_throughput_per_serving_gpu" } +frontiers = [ + { name = "Version A", cases = ["a-c16", "a-c32", "a-c64"] }, + { name = "Version B", cases = ["b-grid"] }, +] +``` + +Here, Version A combines three cases into one frontier; Version B uses every successful point from one sweep case. Combining both versions into a single group would instead select the best points across both versions. + +`direction = "minimize"` or `direction = "maximize"` overrides which values are preferred on an axis. Both axes retain ascending numeric scales. Use the default directions listed above for these inference metrics. + + + +## Viewing and Regenerating Reports + +Charts initially show frontier points. Enable **Show dominated points** to inspect the other measurements. Click a point to pin its details for selection and copying, including inference settings and the executed TOML. The measurements table includes every valid execution, including ties. Lines connect measured points; they do not represent additional measurements between those points. + +To change plots without rerunning workloads, edit the scenario and regenerate reports from saved results: + +``` bash +cloudai generate-report --system-config system.toml --tests-dir tests/ \ + --test-scenario scenario.toml --result-dir results/my-scenario +``` diff --git a/doc/pareto-frontiers.rst b/doc/pareto-frontiers.rst deleted file mode 100644 index 2b2e5eb37..000000000 --- a/doc/pareto-frontiers.rst +++ /dev/null @@ -1,149 +0,0 @@ -Pareto Frontiers -================ - -The ``pareto`` scenario report writes ``pareto.html`` in the scenario results directory. Each plot compares two -metrics. Successful iterations and sweep steps supply the points: one test case can contribute several points, -and several cases can contribute to one frontier. - -Usage Example -------------- - -For :doc:`vLLM ` or :doc:`SGLang `, add this block to your scenario TOML: - -.. code-block:: toml - - [reports.pareto] - enable = true - - [[reports.pareto.plots]] - name = "Streaming latency vs throughput" - x = { metric = "tpot", match = { statistic = "p99" } } - y = { metric = "output_token_throughput" } - frontiers = [{ name = "All configurations" }] - -This selects P99 time per output token on X and total output tokens/s on Y. CloudAI minimizes latency and -maximizes throughput using the metrics' default directions. Both axes increase normally. - -``frontiers`` defines the comparison groups in the plot. Omitting ``cases`` or setting ``cases = []`` includes -all cases in the scenario. Use this example when all cases belong to the same comparison. - -Available Metrics ------------------ - -Use the metric keys below in either ``x.metric`` or ``y.metric``. Each axis must select exactly one measurement -from each execution. ``match`` selects among measurements of the same metric, such as different statistics or -message sizes. Missing or ambiguous measurements are listed as exclusions in the report. - -Both ``vllm`` and ``sglang`` expose these metrics: - -.. list-table:: - :header-rows: 1 - :widths: 65 20 15 - - * - Metric - - Unit - - Default direction - * - ``ttft`` - - ms - - minimize - * - ``tpot`` - - ms - - minimize - * - ``output_token_throughput`` - - tokens/s - - maximize - * - ``request_throughput`` - - requests/s - - maximize - * - ``output_token_throughput_per_allocated_gpu`` - - tokens/s/GPU - - maximize - * - ``output_token_throughput_per_serving_gpu`` - - tokens/s/GPU - - maximize - * - ``accuracy`` - - ratio - - maximize - -``ttft`` measures the wait for the first token; ``tpot`` measures time per output token after the first token. -For either metric, select ``mean``, ``median``, or ``p99`` with ``match = { statistic = "p99" }``. The throughput -metrics have no statistic selector. ``accuracy`` is available when semantic validation is configured and its -result is present; see the workload's Semantic Validation section. - -The two GPU-normalized metrics divide total output tokens/s by different counts: - -- ``output_token_throughput_per_allocated_gpu`` uses the execution's ``num_nodes`` times the system's - ``gpus_per_node``. It includes idle GPUs on those nodes. A larger single-sbatch reservation does not change - the individual execution's count. -- ``output_token_throughput_per_serving_gpu`` uses the configured serving layout, summing prefill and decode - GPUs when both roles are configured. It describes configured devices, not measured GPU utilization. - -Choose the denominator that matches your comparison and use it consistently. A normalized metric is omitted -when its GPU count cannot be determined. - -Examples and Best Practices ---------------------------- - -For inference, start with one of these comparisons: - -- **First-token wait vs capacity:** ``ttft`` on X, ``output_token_throughput`` on Y. -- **Streaming latency vs serving efficiency:** ``tpot`` on X, ``output_token_throughput_per_serving_gpu`` on Y. -- **Streaming latency vs allocated GPU efficiency:** ``tpot`` on X, - ``output_token_throughput_per_allocated_gpu`` on Y. - -Keep the model, input/output token lengths, precision, and traffic pattern comparable. Vary concurrency or -serving parameters to measure the tradeoff. Use P99 for tail latency, or select mean or median when that is -the statistic your comparison requires. Match the statistic and GPU denominator when comparing with another -benchmark report. - -Multiple Plots -~~~~~~~~~~~~~~ - -Append another ``[[reports.pareto.plots]]`` block to generate a separate chart. For example, add a first-token -latency plot alongside the streaming-latency example above: - -.. code-block:: toml - - [[reports.pareto.plots]] - name = "First-token latency vs throughput" - x = { metric = "ttft", match = { statistic = "p99" } } - y = { metric = "output_token_throughput" } - frontiers = [{ name = "All configurations" }] - -Comparing Versions or Configurations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Give each version its own frontier so its measurements compete independently. Append this plot to an enabled -inference report, replacing the ``cases`` entries with your scenario's ``[[Tests]].id`` values: - -.. code-block:: toml - - [[reports.pareto.plots]] - name = "Streaming efficiency by version" - x = { metric = "tpot", match = { statistic = "p99" } } - y = { metric = "output_token_throughput_per_serving_gpu" } - frontiers = [ - { name = "Version A", cases = ["a-c16", "a-c32", "a-c64"] }, - { name = "Version B", cases = ["b-grid"] }, - ] - -Here, Version A combines three cases into one frontier; Version B uses every successful point from one sweep -case. Combining both versions into a single group would instead select the best points across both versions. - -``direction = "minimize"`` or ``direction = "maximize"`` overrides which values are preferred on an axis. -Both axes retain ascending numeric scales. Use the default directions listed above for these inference metrics. - -Viewing and Regenerating Reports --------------------------------- - -Charts initially show frontier points. Enable **Show dominated points** to inspect the other measurements. -Click a point to pin its details for selection and copying, including inference settings and the executed TOML. -The measurements table includes every valid execution, including ties. Lines connect measured points; they do -not represent additional measurements between those points. - -To change plots without rerunning workloads, edit the scenario and regenerate reports from saved results: - -.. code-block:: bash - - cloudai generate-report --system-config system.toml --tests-dir tests/ \ - --test-scenario scenario.toml --result-dir results/my-scenario diff --git a/doc/reporting.mdx b/doc/reporting.mdx new file mode 100644 index 000000000..eae1936cb --- /dev/null +++ b/doc/reporting.mdx @@ -0,0 +1,369 @@ + + +This chapter describes the reporting system in CloudAI. In this chapter, we will cover the following topics: + +- [Overview](reporting.mdx#overview) +- [General Flow](reporting.mdx#general-flow) +- [Enabling, Disabling and Configuring Reports](reporting.mdx#enabling-disabling-and-configuring-reports) +- [Reporting Registration](reporting.mdx#reporting-registration) +- [Reporting Configuration Implementation](reporting.mdx#reporting-configuration-implementation) +- [Uploading Results to Object Storage](reporting.mdx#uploading-results-to-object-storage) +- [Pareto frontiers](pareto-frontiers.mdx) +- [Reports](reports.mdx) + + + +**Overview** + +CloudAI has two reporting levels: + +- per-test (per each case in a test scenario) +- per-scenario (per each test scenario) + +All reports are generated after the test scenario is completed as part of the main CloudAI process. For Slurm, this means that the login node is used to generate reports. + +Per-test reports are linked to a particular workload type (e.g. `NcclTest`). All per-test reports are implemented as part of the `per_test` scenario report and can be enabled or disabled via a single configuration option; see [Enabling, Disabling and Configuring Reports](reporting.mdx#enabling-disabling-and-configuring-reports). + +To list all available reports, users can use `cloudai list-reports`. Use verbose output to also print report configurations. + + + +## Unified Experiment Output + +CloudAI writes `experiment.json` in each scenario's results directory. + +The file contains scenario details, system name, the username of the account running CloudAI in `user`, and test cases listed under `tests`. Older files without `user` are loaded with an empty string. For Slurm runs, the system name is the cluster reported by Slurm. Both Standalone and Slurm execution records are stored in each test case's `runs` list. Each record represents an iteration or DSE step and includes its number, process or Slurm job ID, status, timing, result path, and workload metrics. + +If username lookup fails, CloudAI logs a warning and records an empty `user` without interrupting execution. + +Timestamps use UTC; durations use seconds. Unknown timestamps are `null`. A status of `unknown` indicates the outcome could not be determined. Dry runs include scenario and test-case details without launching workloads. + +Live `runs` status tracking: + +- Standalone jobs are `running` from submission. +- Slurm jobs (excluding single-sbatch) start as `pending` while waiting for allocation, then transition to `running` once allocated and executing. +- Single-sbatch Slurm run records remain `pending` until the allocation finishes and their final statuses are recorded. + +Metrics are sourced from `TestDefinition.metric_observations()`, and are not affected by reporter settings. + +MegatronRun exposes `iteration_time` (milliseconds, lower is better) and `tflops_per_gpu` (TFLOP/s/GPU, higher is better). Each observation is the mean of the last ten iteration metric lines in `stdout.txt`, or all available lines when fewer than ten are present, matching the per-test CSV report. No observations are emitted when iteration metric lines are missing. + +When a test case is executed successfully, `tests[].metrics` contains that execution's metrics. For DSE, it contains metrics from the successful step with the highest valid reward. The search space, selected step, and configuration appear in `tests[].dse`. + + + +### NCCL DSE Example + +This example comes from a Slurm NCCL all-reduce run on one node with eight H100 GPUs. DSE tried `Ring` and `Tree` in two steps. `Ring` (step 1) won using inverse latency as the reward, so its measurements appear in `tests[].metrics`. + +Identifiers and paths are anonymized. To keep the example small, each metrics list includes only out-of-place latency and bus bandwidth for 1 MiB messages. Measurement values, timing, and DSE selection are unchanged. + +`Download experiment.json `. + +``` json +{ + "id": "nccl-dse_2026-09-21_14-55-45", + "name": "nccl-dse", + "system_name": "example-cluster", + "user": "example-user", + "description": null, + "status": "completed", + "path": "/results/nccl-dse_2026-09-21_14-55-45", + "start": "2026-09-21T14:55:45.275325Z", + "finish": "2026-09-21T15:01:50.432934Z", + "duration": 365.157609, + "tests": [ + { + "id": "nccl", + "name": "nccl_base_test", + "description": "NCCL base test configuration", + "status": "completed", + "path": "/results/nccl-dse_2026-09-21_14-55-45/nccl", + "metrics": [ + { + "name": "Latency", + "value": 24.76, + "unit": "us", + "dimensions": [ + {"name": "Operation", "value": "all_reduce", "unit": "", "is_x": false}, + {"name": "Placement", "value": "out_of_place", "unit": "", "is_x": false}, + {"name": "Size", "value": "1048576", "unit": "", "is_x": true} + ] + }, + { + "name": "Bandwidth", + "value": 74.12, + "unit": "GB/s", + "dimensions": [ + {"name": "Bandwidth basis", "value": "bus", "unit": "", "is_x": false}, + {"name": "Operation", "value": "all_reduce", "unit": "", "is_x": false}, + {"name": "Placement", "value": "out_of_place", "unit": "", "is_x": false}, + {"name": "Size", "value": "1048576", "unit": "", "is_x": true} + ] + } + ], + "runs": [ + { + "path": "/results/nccl-dse_2026-09-21_14-55-45/nccl/0/1", + "jobid": "1001", + "status": "completed", + "metrics": [ + { + "name": "Latency", + "value": 24.76, + "unit": "us", + "dimensions": [ + {"name": "Operation", "value": "all_reduce", "unit": "", "is_x": false}, + {"name": "Placement", "value": "out_of_place", "unit": "", "is_x": false}, + {"name": "Size", "value": "1048576", "unit": "", "is_x": true} + ] + }, + { + "name": "Bandwidth", + "value": 74.12, + "unit": "GB/s", + "dimensions": [ + {"name": "Bandwidth basis", "value": "bus", "unit": "", "is_x": false}, + {"name": "Operation", "value": "all_reduce", "unit": "", "is_x": false}, + {"name": "Placement", "value": "out_of_place", "unit": "", "is_x": false}, + {"name": "Size", "value": "1048576", "unit": "", "is_x": true} + ] + } + ], + "start": "2026-09-21T14:55:46Z", + "finish": "2026-09-21T14:57:28Z", + "duration": 102.0, + "iteration": 0, + "step": 1 + }, + { + "path": "/results/nccl-dse_2026-09-21_14-55-45/nccl/0/2", + "jobid": "1002", + "status": "completed", + "metrics": [ + { + "name": "Latency", + "value": 37.36, + "unit": "us", + "dimensions": [ + {"name": "Operation", "value": "all_reduce", "unit": "", "is_x": false}, + {"name": "Placement", "value": "out_of_place", "unit": "", "is_x": false}, + {"name": "Size", "value": "1048576", "unit": "", "is_x": true} + ] + }, + { + "name": "Bandwidth", + "value": 49.11, + "unit": "GB/s", + "dimensions": [ + {"name": "Bandwidth basis", "value": "bus", "unit": "", "is_x": false}, + {"name": "Operation", "value": "all_reduce", "unit": "", "is_x": false}, + {"name": "Placement", "value": "out_of_place", "unit": "", "is_x": false}, + {"name": "Size", "value": "1048576", "unit": "", "is_x": true} + ] + } + ], + "start": "2026-09-21T14:58:47Z", + "finish": "2026-09-21T15:00:28Z", + "duration": 101.0, + "iteration": 0, + "step": 2 + } + ], + "dse": { + "space": { + "extra_env_vars.NCCL_ALGO": ["Ring", "Tree"] + }, + "best_config": {"extra_env_vars.NCCL_ALGO": "Ring"}, + "best_step": 1 + } + } + ] +} +``` + +Each file update replaces the previous snapshot atomically. Metric extraction or write errors produce warnings without affecting execution. + + + +## General Flow + +- All reports should be registered via `Registry()` (`.add_report()` or `.add_scenario_report()`) +- Scenario reports are configurable via system config (Slurm-only for now) and scenario config +- Configuration in a scenario config has the highest priority. Next, system config is checked. Then it defaults to report config from the registry +- Finally, the report is generated (or not) according to this final config + + + +## Enabling, Disabling and Configuring Reports + + + +Only scenario-level reports can be configured. + + + +Enabling or disabling a report needs to be done in the system configuration: + +``` toml +[reports] +per_test = { enable = false } +status = { enable = true } +junit = { enable = true } +``` + +The `junit` scenario reporter is disabled by default. When enabled, it writes `junit.xml` in the scenario results directory. It emits one test case for every regular test iteration and every DSE step, including pass/fail status, failure details, scheduler duration when available, and the contents of `stdout.txt` and `stderr.txt`. The artifact can be consumed directly by Jenkins, GitLab, GitHub Actions, and other CI systems that support JUnit XML. + + + +## Speed-of-Light comparisons + +CloudAI can compare NCCL and NIXL Bench measurements with configured Speed-of-Light (SOL) targets. Targets are validated while the system or scenario TOML is parsed. A target without `match` is the default; a matching target with more dimensions takes precedence: + +``` toml +[[sol.bandwidth]] +value = 100.0 # GB/s + +[[sol.bandwidth]] +value = 120.0 # GB/s +match = { operation = "write", size_bytes = 1048576 } + +[[sol.latency]] +value = 8.0 # us +``` + +NCCL targets can distinguish the collective operation and placement: + +``` toml +[[sol.bandwidth]] +value = 250.0 # GB/s +match = { operation = "all_reduce", placement = "out_of_place", bandwidth_basis = "bus" } + +[[sol.bandwidth]] +value = 300.0 # GB/s +match = { operation = "all_reduce", placement = "in_place", bandwidth_basis = "bus" } +``` + +A test case can replace the targets for a metric inherited from the scenario or system: + +``` toml +[[Tests]] +id = "nixl-case" +test_name = "nixl-bench" + +[[Tests.sol.bandwidth]] +value = 120.0 +``` + +The precedence is test case, then scenario, then system. NCCL and NIXL comparison v2 reports include measured, SOL, and percentage-of-SOL columns and draw a shared SOL curve when every compared run resolves the same targets. + + + +## Reporting Registration + +Report registration is done via `Registry` class: + +``` python +Registry().add_scenario_report("per_test", PerTestReporter, ReportConfig(enable=True)) +``` + + + +## Reporting Configuration Implementation + +Each report can define its own configuration, which is constructed and passed as an argument to `Registry.add_scenario_report`. The `reports` field is parsed during TOML reading and the respective Pydantic model is created for it. + +For example, a custom report configuration can be defined as follows: + +``` python +class CustomReportConfig(ReportConfig): + greeting: str +``` + +``` python +Registry().add_scenario_report("custom", CustomReport, CustomReportConfig(greeting="default value")) +``` + +And it can be used in a test scenario as follows: + +``` toml +[reports] +custom = { enable = true, greeting = "Hello, world!" } +``` + + + +## Uploading Results to Object Storage + +The `s3` scenario report publishes the scenario results directory to an S3-compatible bucket. It is disabled by default, because shipping results off-box should be a deliberate choice. + +It is registered last, after `tarball`, so it always observes the complete results directory including every other report's output. + +Files are uploaded under `s3:///////`, where `cluster` is the system name and `username` is the original experiment owner's `user` field in `experiment.json`, including when running `generate-report`. If the file is missing, unreadable, invalid, or has no recorded owner, uploads use the current account's username. If that lookup also fails, uploads use `unknown`. Tarballs use the same path, with `.tgz` appended. + +Install the optional dependency first: + +``` bash +pip install 'cloudai[s3]' +``` + +Then enable it in a test scenario: + +``` toml +[reports] +s3 = { enable = true, bucket = "my-bucket", prefix = "cloudai/runs", upload_tarball = true } +``` + +Or, for Slurm systems, once per cluster in the system config: + +``` toml +[reports] +s3 = { enable = true, bucket = "my-bucket" } +``` + +Configuration options: + +| Option | Default | Description | +|----|----|----| +| `bucket` | `$CLOUDAI_S3_BUCKET` | Destination bucket. Required; the upload is skipped with a warning if unset. | +| `prefix` | `$CLOUDAI_S3_PREFIX` | Key prefix. Objects are written under `////`. | +| `endpoint_url` | `$CLOUDAI_S3_ENDPOINT_URL` | Custom endpoint, for MinIO or other S3-compatible stores. | +| `region` | unset | AWS region. When unset, boto3 resolves it (e.g. `AWS_DEFAULT_REGION`). | +| `upload_tree` | `true` | Upload each file individually, preserving relative paths. | +| `upload_tarball` | `false` | Also upload a `.tgz` of the whole directory. It is created if absent and rebuilt if older than the results. | +| `upload_concurrency` | `8` | Number of files uploaded concurrently when `upload_tree` is enabled. Must be at least 1. | + +At least one of `upload_tree` and `upload_tarball` must be enabled; the config is rejected otherwise. + + + +### Environment variables + +The destination can be supplied through three environment variables, so a cluster-wide default does not have to be repeated in every scenario: + +| Variable | Sets option | Notes | +|----|----|----| +| `CLOUDAI_S3_BUCKET` | `bucket` | Destination bucket. | +| `CLOUDAI_S3_PREFIX` | `prefix` | Key prefix. Empty by default. | +| `CLOUDAI_S3_ENDPOINT_URL` | `endpoint_url` | Custom endpoint, for MinIO or other S3-compatible stores. An empty value is treated as unset. | + +A value set in TOML takes precedence over the environment variable. The variables are read by the `cloudai` process when the report configuration is loaded, so export them where `cloudai` runs: + +``` bash +export CLOUDAI_S3_BUCKET=my-bucket +export CLOUDAI_S3_PREFIX=cloudai/runs +export CLOUDAI_S3_ENDPOINT_URL=http://localhost:9000 # only for MinIO or other S3-compatible stores +``` + +With these set, enabling the report only needs `s3 = { enable = true }`. + +**Credentials are never read from CloudAI configuration.** They are resolved by boto3's standard chain: `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, `~/.aws/credentials`, or an instance/IAM role. + +Because reports run inside a `try`/`except`, an upload failure logs a warning and leaves the run's exit status unchanged. + +To upload a results directory from an earlier run, re-run the reports against it: + +``` bash +cloudai generate-report --system-config --tests-dir \ + --test-scenario --result-dir results/_ +``` diff --git a/doc/reporting.rst b/doc/reporting.rst deleted file mode 100644 index 02af0d783..000000000 --- a/doc/reporting.rst +++ /dev/null @@ -1,322 +0,0 @@ -Reporting -========= - -This chapter describes the reporting system in CloudAI. In this chapter, we will cover the following topics: - -- :ref:`Overview ` -- :ref:`General Flow ` -- :ref:`Enabling, Disabling and Configuring Reports ` -- :ref:`Reporting Registration ` -- :ref:`Reporting Configuration Implementation ` -- :ref:`Uploading Results to Object Storage ` -- :doc:`Pareto frontiers ` -- :doc:`Reports ` - -.. toctree:: - :hidden: - - pareto-frontiers - reports - -.. _overview: - -**Overview** - -CloudAI has two reporting levels: - -- per-test (per each case in a test scenario) -- per-scenario (per each test scenario) - -All reports are generated after the test scenario is completed as part of the main CloudAI process. For Slurm, this means that the login node is used to generate reports. - -Per-test reports are linked to a particular workload type (e.g. ``NcclTest``). All per-test reports are implemented as part of the ``per_test`` scenario report and can be enabled or disabled via a single configuration option; see :ref:`enabling-disabling-and-configuring-reports`. - -To list all available reports, users can use ``cloudai list-reports``. Use verbose output to also print report configurations. - - -Unified Experiment Output -------------------------- - -CloudAI writes ``experiment.json`` in each scenario's results directory. - -The file contains scenario details, system name, the username of the account running CloudAI in ``user``, and test cases -listed under ``tests``. Older files without ``user`` are loaded with an empty string. For Slurm runs, the system name is -the cluster reported by Slurm. Both Standalone and Slurm execution records are stored in each test case's ``runs`` list. Each -record represents an iteration or DSE step and includes its number, process or Slurm job ID, status, timing, result path, -and workload metrics. - -If username lookup fails, CloudAI logs a warning and records an empty ``user`` without interrupting execution. - -Timestamps use UTC; durations use seconds. Unknown timestamps are ``null``. A status of ``unknown`` indicates the outcome -could not be determined. Dry runs include scenario and test-case details without launching workloads. - -Live ``runs`` status tracking: - -- Standalone jobs are ``running`` from submission. -- Slurm jobs (excluding single-sbatch) start as ``pending`` while waiting for allocation, then transition to ``running`` - once allocated and executing. -- Single-sbatch Slurm run records remain ``pending`` until the allocation finishes and their final statuses are recorded. - -Metrics are sourced from ``TestDefinition.metric_observations()``, and are not affected by reporter settings. - -MegatronRun exposes ``iteration_time`` (milliseconds, lower is better) and ``tflops_per_gpu`` -(TFLOP/s/GPU, higher is better). Each observation is the mean of the last ten iteration metric lines in -``stdout.txt``, or all available lines when fewer than ten are present, matching the per-test CSV report. -No observations are emitted when iteration metric lines are missing. - -When a test case is executed successfully, ``tests[].metrics`` contains that execution's metrics. For DSE, it contains -metrics from the successful step with the highest valid reward. The search space, selected step, and configuration appear -in ``tests[].dse``. - -NCCL DSE Example -~~~~~~~~~~~~~~~~ - -This example comes from a Slurm NCCL all-reduce run on one node with eight H100 GPUs. DSE tried ``Ring`` and ``Tree`` -in two steps. ``Ring`` (step 1) won using inverse latency as the reward, so its measurements appear in ``tests[].metrics``. - -Identifiers and paths are anonymized. To keep the example small, each metrics list includes only out-of-place latency -and bus bandwidth for 1 MiB messages. Measurement values, timing, and DSE selection are unchanged. - -:download:`Download experiment.json `. - -.. literalinclude:: examples/nccl-dse/experiment.json - :language: json - -Each file update replaces the previous snapshot atomically. Metric extraction or write errors produce warnings without -affecting execution. - - -.. _general-flow: - -General Flow ------------- - -- All reports should be registered via ``Registry()`` (``.add_report()`` or ``.add_scenario_report()``) -- Scenario reports are configurable via system config (Slurm-only for now) and scenario config -- Configuration in a scenario config has the highest priority. Next, system config is checked. Then it defaults to report config from the registry -- Finally, the report is generated (or not) according to this final config - - -.. _enabling-disabling-and-configuring-reports: - -Enabling, Disabling and Configuring Reports -------------------------------------------- - -.. note:: - - Only scenario-level reports can be configured. - -Enabling or disabling a report needs to be done in the system configuration: - -.. code-block:: toml - - [reports] - per_test = { enable = false } - status = { enable = true } - junit = { enable = true } - -The ``junit`` scenario reporter is disabled by default. When enabled, it writes ``junit.xml`` in the scenario results -directory. It emits one test case for every regular test iteration and every DSE step, including pass/fail status, -failure details, scheduler duration when available, and the contents of ``stdout.txt`` and ``stderr.txt``. The artifact -can be consumed directly by Jenkins, GitLab, GitHub Actions, and other CI systems that support JUnit XML. - -Speed-of-Light comparisons --------------------------- - -CloudAI can compare NCCL and NIXL Bench measurements with configured Speed-of-Light (SOL) targets. Targets are -validated while the system or scenario TOML is parsed. A target without ``match`` is the default; a matching target -with more dimensions takes precedence: - -.. code-block:: toml - - [[sol.bandwidth]] - value = 100.0 # GB/s - - [[sol.bandwidth]] - value = 120.0 # GB/s - match = { operation = "write", size_bytes = 1048576 } - - [[sol.latency]] - value = 8.0 # us - -NCCL targets can distinguish the collective operation and placement: - -.. code-block:: toml - - [[sol.bandwidth]] - value = 250.0 # GB/s - match = { operation = "all_reduce", placement = "out_of_place", bandwidth_basis = "bus" } - - [[sol.bandwidth]] - value = 300.0 # GB/s - match = { operation = "all_reduce", placement = "in_place", bandwidth_basis = "bus" } - -A test case can replace the targets for a metric inherited from the scenario or system: - -.. code-block:: toml - - [[Tests]] - id = "nixl-case" - test_name = "nixl-bench" - - [[Tests.sol.bandwidth]] - value = 120.0 - -The precedence is test case, then scenario, then system. NCCL and NIXL comparison v2 reports include measured, SOL, -and percentage-of-SOL columns and draw a shared SOL curve when every compared run resolves the same targets. - -.. _reporting-registration: - -Reporting Registration ----------------------- - -Report registration is done via ``Registry`` class: - -.. code-block:: python - - Registry().add_scenario_report("per_test", PerTestReporter, ReportConfig(enable=True)) - -.. _reporting-configuration-implementation: - -Reporting Configuration Implementation ---------------------------------------- - -Each report can define its own configuration, which is constructed and passed as an argument to ``Registry.add_scenario_report``. -The ``reports`` field is parsed during TOML reading and the respective Pydantic model is created for it. - -For example, a custom report configuration can be defined as follows: - -.. code-block:: python - - class CustomReportConfig(ReportConfig): - greeting: str - -.. code-block:: python - - Registry().add_scenario_report("custom", CustomReport, CustomReportConfig(greeting="default value")) - -And it can be used in a test scenario as follows: - -.. code-block:: toml - - [reports] - custom = { enable = true, greeting = "Hello, world!" } - -.. _uploading-results-to-object-storage: - -Uploading Results to Object Storage ------------------------------------- - -The ``s3`` scenario report publishes the scenario results directory to an -S3-compatible bucket. It is disabled by default, because shipping results off-box should -be a deliberate choice. - -It is registered last, after ``tarball``, so it always observes the complete results -directory including every other report's output. - -Files are uploaded under ``s3:///////``, -where ``cluster`` is the system name and ``username`` is the original experiment owner's -``user`` field in ``experiment.json``, including when running ``generate-report``. -If the file is missing, unreadable, invalid, or has no recorded owner, uploads use -the current account's username. If that lookup also fails, uploads use ``unknown``. -Tarballs use the same path, with ``.tgz`` appended. - -Install the optional dependency first: - -.. code-block:: bash - - pip install 'cloudai[s3]' - -Then enable it in a test scenario: - -.. code-block:: toml - - [reports] - s3 = { enable = true, bucket = "my-bucket", prefix = "cloudai/runs", upload_tarball = true } - -Or, for Slurm systems, once per cluster in the system config: - -.. code-block:: toml - - [reports] - s3 = { enable = true, bucket = "my-bucket" } - -Configuration options: - -.. list-table:: - :header-rows: 1 - - * - Option - - Default - - Description - * - ``bucket`` - - ``$CLOUDAI_S3_BUCKET`` - - Destination bucket. Required; the upload is skipped with a warning if unset. - * - ``prefix`` - - ``$CLOUDAI_S3_PREFIX`` - - Key prefix. Objects are written under ``////``. - * - ``endpoint_url`` - - ``$CLOUDAI_S3_ENDPOINT_URL`` - - Custom endpoint, for MinIO or other S3-compatible stores. - * - ``region`` - - unset - - AWS region. When unset, boto3 resolves it (e.g. ``AWS_DEFAULT_REGION``). - * - ``upload_tree`` - - ``true`` - - Upload each file individually, preserving relative paths. - * - ``upload_tarball`` - - ``false`` - - Also upload a ``.tgz`` of the whole directory. It is created if absent and rebuilt if older than the results. - * - ``upload_concurrency`` - - ``8`` - - Number of files uploaded concurrently when ``upload_tree`` is enabled. Must be at least 1. - -At least one of ``upload_tree`` and ``upload_tarball`` must be enabled; the config is rejected otherwise. - -Environment variables -~~~~~~~~~~~~~~~~~~~~~ - -The destination can be supplied through three environment variables, so a cluster-wide -default does not have to be repeated in every scenario: - -.. list-table:: - :header-rows: 1 - - * - Variable - - Sets option - - Notes - * - ``CLOUDAI_S3_BUCKET`` - - ``bucket`` - - Destination bucket. - * - ``CLOUDAI_S3_PREFIX`` - - ``prefix`` - - Key prefix. Empty by default. - * - ``CLOUDAI_S3_ENDPOINT_URL`` - - ``endpoint_url`` - - Custom endpoint, for MinIO or other S3-compatible stores. An empty value is treated as unset. - -A value set in TOML takes precedence over the environment variable. The variables are read by -the ``cloudai`` process when the report configuration is loaded, so export them where -``cloudai`` runs: - -.. code-block:: bash - - export CLOUDAI_S3_BUCKET=my-bucket - export CLOUDAI_S3_PREFIX=cloudai/runs - export CLOUDAI_S3_ENDPOINT_URL=http://localhost:9000 # only for MinIO or other S3-compatible stores - -With these set, enabling the report only needs ``s3 = { enable = true }``. - -**Credentials are never read from CloudAI configuration.** They are resolved by boto3's -standard chain: ``AWS_ACCESS_KEY_ID``/``AWS_SECRET_ACCESS_KEY``, ``~/.aws/credentials``, -or an instance/IAM role. - -Because reports run inside a ``try``/``except``, an upload failure logs a warning and -leaves the run's exit status unchanged. - -To upload a results directory from an earlier run, re-run the reports against it: - -.. code-block:: bash - - cloudai generate-report --system-config --tests-dir \ - --test-scenario --result-dir results/_ diff --git a/doc/reports.rst b/doc/reports.mdx similarity index 53% rename from doc/reports.rst rename to doc/reports.mdx index dca865a1c..504851875 100644 --- a/doc/reports.rst +++ b/doc/reports.mdx @@ -1,9 +1,5 @@ -Reports -======= + CloudAI report schemas describe the generated output for supported workload categories. -.. toctree:: - :maxdepth: 1 - - training-report-schema +- [Training Report Schema](training-report-schema.mdx) diff --git a/doc/systems.mdx b/doc/systems.mdx new file mode 100644 index 000000000..93687e450 --- /dev/null +++ b/doc/systems.mdx @@ -0,0 +1,53 @@ + + +This chapter lists all the systems supported by CloudAI. The attributes shown for each system can be set in TOML configuration files. + +| System | Scheduler Value | +|---------------------------------------------|-----------------| +| [Slurm](systems.mdx#slurm-system) | `slurm` | +| [Kubernetes](systems.mdx#kubernetes-system) | `kubernetes` | +| [RunAI](systems.mdx#runai-system) | `runai` | +| [LSF](systems.mdx#lsf-system) | `lsf` | +| [Standalone](systems.mdx#standalone-system) | `standalone` | + + + +## Slurm + + + + + + + + + + + +## Kubernetes + + + + + +## RunAI + + + + + + + +## LSF + + + + + + + + + +## Standalone + + diff --git a/doc/systems.rst b/doc/systems.rst deleted file mode 100644 index a76ab87fa..000000000 --- a/doc/systems.rst +++ /dev/null @@ -1,67 +0,0 @@ -Systems -======= - -This chapter lists all the systems supported by CloudAI. The attributes shown for each system can be set in TOML configuration files. - -.. csv-table:: - :header: "System", "Scheduler Value" - :widths: 40, 30 - - ":ref:`slurm-system`", "``slurm``" - ":ref:`kubernetes-system`", "``kubernetes``" - ":ref:`runai-system`", "``runai``" - ":ref:`lsf-system`", "``lsf``" - ":ref:`standalone-system`", "``standalone``" - -.. _slurm-system: - -Slurm ------ - -.. autopydantic_model:: cloudai.systems.slurm.slurm_system.SlurmSystem - :exclude-members: cmd_shell, group_allocated, supports_gpu_directives_cache - -.. autopydantic_model:: cloudai.systems.slurm.slurm_system.SlurmPartition - :exclude-members: slurm_nodes - -.. autopydantic_model:: cloudai.systems.slurm.slurm_system.SlurmGroup - -.. autopydantic_model:: cloudai.systems.slurm.slurm_system.DataRepositoryConfig - -.. _kubernetes-system: - -Kubernetes ----------- - -.. autopydantic_model:: cloudai.systems.kubernetes.kubernetes_system.KubernetesSystem - -.. _runai-system: - -RunAI ------ - -.. autopydantic_model:: cloudai.systems.runai.runai_system.RunAISystem - :exclude-members: nodes - -.. autopydantic_model:: cloudai.systems.runai.runai_node.RunAINode - -.. _lsf-system: - -LSF ---- - -.. autopydantic_model:: cloudai.systems.lsf.lsf_system.LSFSystem - :exclude-members: cmd_shell - -.. autopydantic_model:: cloudai.systems.lsf.lsf_system.LSFQueue - :exclude-members: lsf_nodes - -.. autopydantic_model:: cloudai.systems.lsf.lsf_system.LSFGroup - -.. _standalone-system: - -Standalone ----------- - -.. autopydantic_model:: cloudai.systems.standalone.standalone_system.StandaloneSystem - :exclude-members: cmd_shell diff --git a/doc/templates/reference.mdx.jinja b/doc/templates/reference.mdx.jinja new file mode 100644 index 000000000..f5af5aab2 --- /dev/null +++ b/doc/templates/reference.mdx.jinja @@ -0,0 +1,56 @@ + + +### `{{ symbol }}` + +{{ description | mdx }} +{% for field in fields %} + + +#### `{{ field.name }}` + +```python +{{ field.name }}: {{ field.annotation }}{% if field.default is not none %} = {{ field.default }}{% endif %} +``` + +{% if field.required %}**Required.**{% elif field.factory %}**Default factory:** `{{ field.factory }}`.{% endif %} +{% if field.alias %} +**Alias:** `{{ field.alias }}`. +{% endif %} +{% if field.constraints %} +**Constraints:** {% for constraint in field.constraints %}`{{ constraint }}`{% if not loop.last %}, {% endif %}{% endfor %}. +{% endif %} + +{{ field.description | mdx }} +{% endfor %} +{% for member in members %} + + +#### `{{ member.name }}` + +```python +{{ member.name }}{{ member.signature }} +``` +{% for section in member.sections %} +{% if section.kind.value == 'text' %} +{{ section.value | mdx }} +{% elif section.kind.value in ['parameters', 'other parameters', 'attributes', 'returns', 'yields', 'raises', 'warns'] %} +**{{ section.title or section.kind.value | title }}:** +{% for item in section.value %} +- {% if item.name is defined and item.name %}`{{ item.name }}`{% endif %}{% if item.annotation %} (`{{ item.annotation }}`){% endif %}: {{ item.description | mdx }} +{% endfor %} +{% elif section.kind.value == 'examples' %} +**Examples:** +{% for kind, text in section.value %} +{% if kind.value == 'examples' %} +```python +{{ text }} +``` +{% else %} +{{ text | mdx }} +{% endif %} +{% endfor %} +{% elif section.kind.value == 'admonition' %} +{{ section.value.description | mdx }} +{% endif %} +{% endfor %} +{% endfor %} diff --git a/doc/training-report-schema.mdx b/doc/training-report-schema.mdx new file mode 100644 index 000000000..16aef41a8 --- /dev/null +++ b/doc/training-report-schema.mdx @@ -0,0 +1,60 @@ + + +`training_report.json` is the unified training output for NeMoRun, MegatronRun, and Megatron-Bridge. + + + +## Versioning + +The top-level `schema_version` field uses `MAJOR.MINOR` versioning: + +- Increment **MAJOR** when removing, renaming, or incompatibly changing a field. +- Increment **MINOR** when adding an optional field or otherwise making a backward-compatible schema change. +- Do not change the schema version for implementation fixes that leave the JSON contract unchanged. + +Consumers should reject unsupported major versions and tolerate unknown fields within a supported major version. + + + +## Version History + + + +### 1.0 — 2026-07-14 + +Added to `root`: + +- `schema_version`: `str` + +Added to `root.config`: + +- `test_id`: `str` +- `test_name`: `str` +- `description`: `str` +- `test_scenario_name`: `str` +- `system_path`: `str` +- `tests_dir_path`: `str` +- `test_scenario_path`: `str` +- `container_image`: `str` +- `cloudai_execution_node`: `str` +- `env_vars`: `dict[str, Any]` +- `gpus_per_node`: `Optional[int]` +- `nodes`: `list[str]` +- `clique_size`: `Optional[int]` +- `fp8`: `Optional[str]` +- `fp8_recipe`: `Optional[str]` +- `expert_tensor_parallel_size`: `int` + + + +## Training Report Models + + + + + + + + + + diff --git a/doc/training-report-schema.rst b/doc/training-report-schema.rst deleted file mode 100644 index f589890a2..000000000 --- a/doc/training-report-schema.rst +++ /dev/null @@ -1,67 +0,0 @@ -Training Report Schema -====================== - -``training_report.json`` is the unified training output for NeMoRun, MegatronRun, and Megatron-Bridge. - -Versioning ----------- - -The top-level ``schema_version`` field uses ``MAJOR.MINOR`` versioning: - -- Increment **MAJOR** when removing, renaming, or incompatibly changing a field. -- Increment **MINOR** when adding an optional field or otherwise making a backward-compatible schema change. -- Do not change the schema version for implementation fixes that leave the JSON contract unchanged. - -Consumers should reject unsupported major versions and tolerate unknown fields within a supported major version. - -Version History ---------------- - -1.0 — 2026-07-14 -~~~~~~~~~~~~~~~~ - -Added to ``root``: - -- ``schema_version``: ``str`` - -Added to ``root.config``: - -- ``test_id``: ``str`` -- ``test_name``: ``str`` -- ``description``: ``str`` -- ``test_scenario_name``: ``str`` -- ``system_path``: ``str`` -- ``tests_dir_path``: ``str`` -- ``test_scenario_path``: ``str`` -- ``container_image``: ``str`` -- ``cloudai_execution_node``: ``str`` -- ``env_vars``: ``dict[str, Any]`` -- ``gpus_per_node``: ``Optional[int]`` -- ``nodes``: ``list[str]`` -- ``clique_size``: ``Optional[int]`` -- ``fp8``: ``Optional[str]`` -- ``fp8_recipe``: ``Optional[str]`` -- ``expert_tensor_parallel_size``: ``int`` - -Training Report Models ----------------------- - -.. autoclass:: cloudai.report_generator.training.models.TrainingResults - :members: - :exclude-members: __init__ - -.. autoclass:: cloudai.report_generator.training.models.TrainingConfig - :members: - :exclude-members: __init__ - -.. autoclass:: cloudai.report_generator.training.models.TrainingStep - :members: - :exclude-members: __init__ - -.. autoclass:: cloudai.report_generator.training.models.StepAggregation - :members: - :exclude-members: __init__, from_steps - -.. autoclass:: cloudai.report_generator.training.models.MetricStats - :members: - :exclude-members: __init__, from_values diff --git a/doc/troubleshooting.mdx b/doc/troubleshooting.mdx new file mode 100644 index 000000000..c9cc264e5 --- /dev/null +++ b/doc/troubleshooting.mdx @@ -0,0 +1,28 @@ + + +This section guides you through identifying the root cause of issues, determining whether they stem from system infrastructure or a bug in CloudAI. + + + +## Identifying the Root Cause + +If you encounter issues running a command, start by reading the error message to understand the root cause. We strive to make our error messages and exception messages as readable and interpretable as possible. + + + +## System Infrastructure vs. CloudAI Bugs + +To determine whether an issue is due to system infrastructure or a CloudAI bug, follow these steps: + +1. **Check stdout Messages:** If CloudAI fails to run a test successfully, it will be indicated in the stdout messages that a test has failed. +2. **Review Log Files:** + - Navigate to the output directory and review `debug.log`, stdout, and stderr files + - `debug.log` contains detailed steps executed by CloudAI, including generated commands, executed commands, and error messages +3. **Analyze Error Messages:** By examining the error messages in the log files, you can understand the type of errors CloudAI encountered. +4. **Examine Output Directory:** If a test fails without explicit error messages, review the output directory of the failed test. Look for `stdout.txt`, `stderr.txt`, or any generated files to understand the failure reason. +5. **Manual Rerun of Tests:** + - To manually rerun the test, consult the `debug.log` for the command CloudAI executed + - Look for an `sbatch` command with a generated `sbatch` script + - Execute the command manually to debug further + +If the problem persists, please report the issue at [https://github.com/NVIDIA/cloudai/issues/new/choose](https://github.com/NVIDIA/cloudai/issues/new/choose). When you report an issue, please make sure it is reproducible. Follow the issue template and provide any necessary details, such as the hash commit used, system settings, any changes in the schema files, and the command. diff --git a/doc/troubleshooting.rst b/doc/troubleshooting.rst deleted file mode 100644 index 237b32c6a..000000000 --- a/doc/troubleshooting.rst +++ /dev/null @@ -1,33 +0,0 @@ -Troubleshooting -=============== - -This section guides you through identifying the root cause of issues, determining whether they stem from system infrastructure or a bug in CloudAI. - -Identifying the Root Cause ---------------------------- - -If you encounter issues running a command, start by reading the error message to understand the root cause. We strive to make our error messages and exception messages as readable and interpretable as possible. - -System Infrastructure vs. CloudAI Bugs ---------------------------------------- - -To determine whether an issue is due to system infrastructure or a CloudAI bug, follow these steps: - -1. **Check stdout Messages:** If CloudAI fails to run a test successfully, it will be indicated in the stdout messages that a test has failed. - -2. **Review Log Files:** - - - Navigate to the output directory and review ``debug.log``, stdout, and stderr files - - ``debug.log`` contains detailed steps executed by CloudAI, including generated commands, executed commands, and error messages - -3. **Analyze Error Messages:** By examining the error messages in the log files, you can understand the type of errors CloudAI encountered. - -4. **Examine Output Directory:** If a test fails without explicit error messages, review the output directory of the failed test. Look for ``stdout.txt``, ``stderr.txt``, or any generated files to understand the failure reason. - -5. **Manual Rerun of Tests:** - - - To manually rerun the test, consult the ``debug.log`` for the command CloudAI executed - - Look for an ``sbatch`` command with a generated ``sbatch`` script - - Execute the command manually to debug further - -If the problem persists, please report the issue at https://github.com/NVIDIA/cloudai/issues/new/choose. When you report an issue, please make sure it is reproducible. Follow the issue template and provide any necessary details, such as the hash commit used, system settings, any changes in the schema files, and the command. diff --git a/doc/workloads/ai_dynamo.mdx b/doc/workloads/ai_dynamo.mdx new file mode 100644 index 000000000..a7f6b1582 --- /dev/null +++ b/doc/workloads/ai_dynamo.mdx @@ -0,0 +1,443 @@ + + +AI Dynamo workload (`test_template_name` is `AIDynamo`) runs AI inference benchmarks using the Dynamo framework with distributed prefill and decode workers. + + + +## Run Using Kubernetes + + + +### Prepare Cluster + +Before running the AI Dynamo workload on a Kubernetes cluster, ensure that the cluster is set up according to the instructions in the [official documentation](https://docs.nvidia.com/dynamo/dev/getting-started/kubernetes-deployment). Below is a short summary of the required steps: + +``` bash +export NAMESPACE=dynamo-system +export RELEASE_VERSION=0.7.0 # replace with the desired release version + +helm upgrade -n default -i dynamo-crds https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-crds-${RELEASE_VERSION}.tgz +helm upgrade -n default -i dynamo-platform https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-${RELEASE_VERSION}.tgz + +# The following components are required for multi node only. +# Versions should be aligned with Dynamo version. +helm upgrade -n default -i grove oci://ghcr.io/ai-dynamo/grove/grove-charts:v0.0.0-gd462e65 +helm upgrade -n default -i kai-scheduler oci://ghcr.io/nvidia/kai-scheduler/kai-scheduler:0.0.0-4c29820 +``` + + + +### Launch and Monitor the Job + + + +Both CloudAI and Dynamo will try to access HuggingFace Hub. To avoid `429 Too Many Requests` errors and access models under auth, it is recommended to define `HF_TOKEN` environment variable before invoking CloudAI. Once set, run `uv run hf auth login` to authenticate. + + + +``` bash +uv run cloudai run --system-config \ + --tests-dir conf/experimental/ai_dynamo/test \ + --test-scenario conf/experimental/ai_dynamo/test_scenario/vllm_k8s.toml +``` + + + +## Run Using Slurm + + + +### Node Configuration for AI Dynamo + +AI Dynamo jobs use three distinct types of nodes: + +- **Frontend node**: Hosts the coordination services (`etcd`, `nats`), the **frontend server**, the **request generator** (`aiperf` by default, configurable via `workloads` in the test TOML), and the first decode worker +- **Prefill node(s)**: Handle the prefill stage of inference +- **Decode node(s)**: Handle the decode stage of inference (optional, depending on model and setup) + +By default, when `num_nodes` is omitted, CloudAI allocates separate nodes for prefill and decode workers: + + num_prefill_nodes + num_decode_nodes + +Set top-level `num_nodes` explicitly to control the Slurm allocation. A value lower than `num_prefill_nodes + num_decode_nodes` enables shared-node disaggregated inference, where prefill and decode roles run on the same allocated node(s) with separate GPU slices. + +All node role assignments and orchestration are automatically managed by CloudAI. + + + +### Multinode Backend Workers on Slurm + +`prefill_worker.num-nodes` and `decode_worker.num-nodes` specify the total nodes assigned to each role. Set `nodes-per-worker` to group those nodes into logical backend workers. For example, the following configuration creates one prefill worker spanning two nodes and one decode worker spanning two other nodes: + +``` toml +[cmd_args.dynamo.prefill_worker] +num-nodes = 2 +nodes-per-worker = 2 + [cmd_args.dynamo.prefill_worker.args] + tensor-parallel-size = 8 + pipeline-parallel-size = 1 + +[cmd_args.dynamo.decode_worker] +num-nodes = 2 +nodes-per-worker = 2 + [cmd_args.dynamo.decode_worker.args] + tensor-parallel-size = 8 + pipeline-parallel-size = 1 +``` + +When `nodes-per-worker` is omitted, it defaults to `1` and preserves the previous one-worker-per-node behavior. For vLLM TP/PP groups, CloudAI launches rank 0 as the Dynamo-facing process and the remaining ranks with `--headless` using Dynamo's multiprocessing executor. When `data-parallel-size` is greater than one, CloudAI instead launches a full Dynamo vLLM process on every node and derives `data-parallel-size-local`, `data-parallel-start-rank`, the coordinator address, and the RPC port from the worker topology. SGLang receives `--dist-init-addr`, `--nnodes`, and `--node-rank` on every participating node; its configured `data-parallel-size` is forwarded as `dp-size`. The launcher derives prefill and decode disaggregation modes from the configured roles, so worker commands do not need role flags. + +For TP/PP-spanning workers, `tensor-parallel-size * pipeline-parallel-size` must be divisible by `nodes-per-worker`. For vLLM multinode data parallelism, `data-parallel-size` must instead be divisible by `nodes-per-worker`, and each DP rank's TP/PP group must fit within one node. SGLang multinode data parallelism requires `--enable-dp-attention`. The resulting per-node GPU footprint must fit on each node. Ray-backed Dynamo vLLM groups are not currently supported. The Slurm launcher explicitly selects etcd discovery and the NATS event plane. + +No separate WideEP mode is required. Multinode data-parallel orchestration is inferred from `nodes-per-worker > 1` and `data-parallel-size > 1`. Backend expert-parallel parameters such as vLLM's `--enable-expert-parallel` or SGLang's `ep-size` and `--enable-dp-attention` remain explicit backend configuration. CloudAI uses the allocated Slurm node names for multinode rendezvous. The cluster must make those names consistently resolvable and mutually reachable from every allocated node and workload container. Multi-homed systems should also configure backend network selection, such as `GLOO_SOCKET_IFNAME` and `NCCL_SOCKET_IFNAME`, at the system level. Explicit backend rendezvous addresses take precedence over addresses derived from the Slurm node names. Clusters that cannot provide this hostname contract must supply backend-native per-node address overrides, such as `SGLANG_HOST_IP` or `VLLM_NIXL_SIDE_CHANNEL_HOST`, together with explicit worker rendezvous arguments. + +Dedicated examples are available in `test_scenario/vllm_multinode_worker_slurm.toml` and `test_scenario/sglang_multinode_worker_slurm.toml`. The existing `vllm_slurm.toml` and `sglang_slurm.toml` scenarios retain their previous configurations. The vLLM example inherits `tensor-parallel-size = 8` from `test/vllm.toml`; the SGLang example sets it explicitly because `test/sglang.toml` defaults to `1`. WideEP examples are provided in `test_scenario/vllm_wideep_slurm.toml` and `test_scenario/sglang_wideep_slurm.toml`. + + + +### Launch and Monitor the Job + +To run the job: + +``` bash +uv run cloudai run --system-config \ + --tests-dir conf/experimental/ai_dynamo/test \ + --test-scenario conf/experimental/ai_dynamo/test_scenario/vllm_slurm.toml +``` + +The job progress monitoring can be done using either of the following options: + +``` bash +watch squeue --me +``` + +``` bash +watch tail -n 4 ./results//*.txt +``` + +The frontend node will initially wait to allow weight loading on all nodes. Once ready, it will launch the configured benchmark tool (`aiperf` by default), which begins generating requests to the frontend server. All servers cooperate to complete inference, and the output will appear in `stdout.txt`. + + + +### Choosing a Benchmark Tool + +The benchmark tool is controlled by the `workloads` field in the test TOML. Set `aiperf.sh` to use AIPerf: + +``` toml +[cmd_args] +workloads = "aiperf.sh" # uses aiperf, writes aiperf_report.csv +``` + +To use genai-perf, set: + +``` toml +[cmd_args] +workloads = "genai_perf.sh" # uses genai-perf, writes genai_perf_report.csv + +[cmd_args.genai_perf] +cmd = "genai-perf profile" +extra-args = "--streaming --verbose -- -v --async" + + [cmd_args.genai_perf.args] + endpoint-type = "chat" + output-tokens-mean = 500 + request-count = 50 +``` + + + +### AIPerf Multi-Phase Runs + +`cmd_args.aiperf` is the base AIPerf config. `cmd_args.aiperf_phases` can run several AIPerf rounds against the same live Dynamo stack. By default, CloudAI does not restart prefill, decode, or router processes between phases: + +``` toml +dse_excluded_args = ["cmd_args.aiperf_phases"] + +[cmd_args.aiperf] +health-check-between-phases = true +between-phase-cmd = "true" # default no-op + + [cmd_args.aiperf.args] + request-count = 50 + server-metrics = "auto" + +[[cmd_args.aiperf_phases]] +name = "round_1" + [cmd_args.aiperf_phases.args] + concurrency = 2 + +[[cmd_args.aiperf_phases]] +name = "round_2" + [cmd_args.aiperf_phases.args] + concurrency = 4 +``` + +Single-phase runs keep the old artifact layout: `aiperf_artifacts/`, `aiperf.log`, and `aiperf_report.csv`. Multi-phase runs write per-phase artifacts/logs/reports and copy the last phase report to `aiperf_report.csv` for existing report generation. + +`between-phase-cmd` is a bash command run after each non-final phase. The default is a no-op. Set it explicitly for backend-specific cache cleanup, for example `/cloudai_run_results/routerctl.sh restart` if a test needs to restart the Dynamo router between phases. `health-check-between-phases` probes the frontend after the command. + + + +## Comparison Reports + +CloudAI provides the scenario-level `ai_dynamo_comparison` report. It compares AI Dynamo runs using the standard LLM serving tables and charts for TTFT, TPOT, successful prompts, output-token throughput, TPS per user, and TPS per GPU. When `aiperf_accuracy` is configured and its result is available, the report also compares model accuracy. Both AIPerf and GenAI-Perf CSV output are supported. + +AIPerf args are rendered as normal CLI flags. Multi-value AIPerf options should be passed with AIPerf CLI syntax, such as `server-metrics-formats = "csv,json,jsonl"` or `gpu-telemetry = "node1:9401,node2:9401"`. `server-metrics = "auto"` expands to the frontend metrics endpoint, Dynamo worker metrics endpoints, and any CloudAI-started DCGM exporters. + + + +### Propagating LMCache Configuration + +AIDynamo can pass an LMCache YAML config to the worker processes by setting `LMCACHE_CONFIG_FILE` inside the container. This only propagates the LMCache configuration; the vLLM/SGLang runtime still needs to be launched with the appropriate LMCache or KV-transfer connector for that image/version. + +The preferred form is structured TOML under `[cmd_args.lmcache]`. CloudAI converts that object to YAML in the run output directory, mounts that directory as `/cloudai_run_results`, and exports the generated file path as `LMCACHE_CONFIG_FILE`: + +``` toml +[cmd_args] + [cmd_args.lmcache] + chunk_size = 256 + local_cpu = true + controller_pull_url = "{frontend_node}:8300" + controller_reply_url = "{frontend_node}:8400" + lmcache_worker_ports = [8788, 8789, 8790, 8791] + max_local_cpu_size = 6.0 + nixl_buffer_size = 2079377920 + nixl_buffer_device = "cpu" + + [cmd_args.lmcache.extra_config] + enable_nixl_storage = false + nixl_backend = "POSIX" + nixl_path = "{storage_cache_dir}" + nixl_pool_size = 2048 +``` + +For an example that uses test-in-scenario mode, see `conf/experimental/ai_dynamo/test_scenario/vllm_lmcache.toml`. Because the test is fully defined inside the scenario, `--tests-dir` is not required when running that example: + +``` bash +uv run cloudai run --system-config \ + --test-scenario conf/experimental/ai_dynamo/test_scenario/vllm_lmcache.toml +``` + +The example sets `dse_excluded_args = ["cmd_args.lmcache.lmcache_worker_ports"]` because `lmcache_worker_ports` is a list-valued LMCache setting, not a DSE sweep dimension. Other list-valued LMCache fields can still be swept unless their `cmd_args.` path is also excluded. + +Alternatively, mount your own LMCache YAML file with `extra_container_mounts` and set `LMCACHE_CONFIG_FILE` through `extra_env_vars`: + +``` toml +extra_container_mounts = ["/host/lmcache:/lmcache"] +extra_env_vars = { LMCACHE_CONFIG_FILE = "/lmcache/config.yaml" } +``` + +For multi-node LMCache storage tests, any path referenced by the LMCache YAML, such as `nixl_path` for POSIX-backed storage, must be visible and writable from every node that is expected to share cached data. A node-local path such as `/tmp` is suitable only for single-node smoke tests or configuration propagation checks. + +LMCache YAML values can use runtime placeholders. CloudAI renders them inside the Slurm job before launching workers: `{frontend_node}`, `{frontend_ip}`, `{results_dir}`, and `{storage_cache_dir}`. Unknown placeholders fail the run before worker processes start. + + + +### Propagating HiCache Configuration + +Define a HiCache storage-backend configuration under `[cmd_args.hicache]`. CloudAI serializes the object as `hicache-config.toml` in the run output directory, exposes its container path through `HICACHE_CONFIG_FILE`, and passes the resolved path to both workers as `--hicache-storage-backend-extra-config`: + +``` toml +[cmd_args] + [cmd_args.hicache.plugin.posix] + active = true + use_uring = "false" + + [cmd_args.dynamo.prefill_worker] + extra-args = "--enable-hierarchical-cache" + + [cmd_args.dynamo.prefill_worker.args] + hicache-size = 8 + hicache-ratio = 0 + hicache-write-policy = "write_through" + hicache-mem-layout = "page_first" + hicache-storage-prefetch-policy = "wait_complete" + hicache-storage-backend = "nixl" + + [cmd_args.dynamo.decode_worker] + extra-args = "--enable-hierarchical-cache" + + [cmd_args.dynamo.decode_worker.args] + hicache-size = 8 + hicache-ratio = 0 + hicache-write-policy = "write_through" + hicache-mem-layout = "page_first" + hicache-storage-prefetch-policy = "wait_complete" + hicache-storage-backend = "nixl" +``` + +The remaining HiCache engine options stay in the worker `args` sections so they can be configured or swept through DSE normally. + +The runtime replacement mechanism below supports per-node HiCache values. For example, change the POSIX setting above to `use_uring = "${use_uring}"` and have the startup manifest replace `${use_uring}` with `false` or `true`. The worker receives the resulting per-node TOML path automatically. + +For a self-contained single-node smoke test using NIXL's POSIX plugin, run the following on a node with at least two GPUs (one each for the prefill and decode workers): + +``` bash +uv run cloudai run --system-config \ + --test-scenario conf/experimental/ai_dynamo/test_scenario/sglang_hicache.toml +``` + + + +### Per-node Startup Commands and Runtime Replacements + +On Slurm, `startup_cmd` runs once per allocated node before the workload. It runs on the host by default. Set `startup_cmd_docker_image` to run it in a container using the workload's mounts: + +``` toml +[cmd_args] +startup_cmd = "/mnt/discovery/discover-device.sh" +startup_cmd_docker_image = "nvcr.io/example/discovery-tools:latest" +``` + +CloudAI exports `CLOUDAI_RUNTIME_FILE` to the command. It points to `runtime/.json` in the test output directory, or `/cloudai_run_results/runtime/.json` inside a startup container. Write replacements there: + +``` json +{ + "LMCACHE_CONFIG_FILE": { + "${device}": "/dev/ng4n1", + "${namespace_id}": "1" + }, + "HICACHE_CONFIG_FILE": { + "${use_uring}": "false" + } +} +``` + +Each top-level key names an environment variable containing a config path. AIDynamo creates a per-node copy, applies literal replacements, then updates the variable to point to that copy. + +For `lmcache-config.yaml` on `node-01`, the result is `lmcache-config.node-01.yaml`. The shared source remains unchanged. LMCache placeholders are rendered afterward in the per-node copy. Missing manifest means no replacements. + +Startup runs once per test case in normal and `--single-sbatch` modes. + +If the selected LMCache mode needs a controller, CloudAI can start one on the frontend node: + +``` toml +[cmd_args.lmcache_controller] +cmd = "lmcache_controller --host 0.0.0.0 --port 9000 --monitor-ports {\"pull\":8300,\"reply\":8400}" +``` + +This only launches the process. For disaggregated or multi-node runs, the LMCache YAML still needs controller addresses that resolve to the frontend node from every worker. With the default controller monitor ports, use `controller_pull_url = "{frontend_node}:8300"` and `controller_reply_url = "{frontend_node}:8400"`. The `lmcache_worker_ports` list must match the number of worker ranks. + + + +### Semantic Degradation With AIPerf Accuracy + +AIDynamo uses AIPerf accuracy mode as its semantic degradation signal. Enable it with `[cmd_args.aiperf_accuracy]`. This runs after the configured performance workload, so it can be used with either `aiperf.sh` or `genai_perf.sh`: + +``` toml +[cmd_args] +workloads = "aiperf.sh" + +[cmd_args.aiperf] + [cmd_args.aiperf.args] + request-count = 50 + synthetic-input-tokens-mean = 300 + output-tokens-mean = 500 + concurrency = 2 + +[cmd_args.aiperf_accuracy] +entrypoint = "aiperf profile" +setup-cmd = "python -m pip install --break-system-packages --upgrade aiperf==0.8.0" +cli = ''' +--model {model} +--url {url} +--endpoint-type chat +--streaming +--artifact-dir {artifact_dir} +--no-server-metrics +--accuracy-benchmark mmlu +--accuracy-n-shots 5 +--accuracy-tasks abstract_algebra +--concurrency 10 +--extra-inputs '{"temperature":0,"chat_template_kwargs":{"enable_thinking":false}}' +--num-requests 100 +''' +``` + +When `cmd_args.aiperf_accuracy` is configured, CloudAI expects AIPerf to produce `accuracy_results.csv` and exposes the `accuracy` metric from its `OVERALL` row. The metric is reported as a 0.0-1.0 fraction. Keep synthetic prompt and token-length flags out of this mode; the benchmark dataset should come from AIPerf's accuracy benchmark. + +The `entrypoint` and `cli` fields form the accuracy command. CloudAI expands `{model}`, `{url}`, `{endpoint}`, `{result_dir}`, and `{artifact_dir}` in `cli` before launching it. The `setup-cmd` field is optional. It is useful for Dynamo images that include an older system `aiperf` build without the accuracy benchmark plugins. The example upgrades the image-level `aiperf` before launching `aiperf profile`. MMLU is loaded from `lighteval/mmlu`, so either allow Hugging Face dataset access or pre-cache that dataset before running with `HF_HUB_OFFLINE`/`HF_DATASETS_OFFLINE` enabled. For Qwen3 models, the example disables thinking mode so short MMLU answers can be parsed as choices. + + + +### Custom Accuracy Scripts + +`cmd_args.aiperf_accuracy` can also launch a custom mounted script instead of AIPerf. Mount the script or its parent directory with `extra_container_mounts` and set `entrypoint` to the in-container command: + +``` toml +extra_container_mounts = ["/host/custom_accuracy:/custom_accuracy"] + +[cmd_args.aiperf_accuracy] +entrypoint = "python /custom_accuracy/dummy_accuracy.py" +cli = "--model {model} --url {url} --endpoint {endpoint} --artifact-dir {artifact_dir} --prompt ping" +``` + +CloudAI expands placeholders in `cli` and runs `entrypoint` with that CLI string. The custom command must write `accuracy_results.csv` inside `{artifact_dir}` with an `OVERALL` row. CloudAI copies that file to the run output directory and exposes the same `accuracy` metric as AIPerf accuracy mode. + + + +## Review Benchmark Results + +After job completion, CloudAI places output logs and result files in the designated results directory. The result file name depends on the configured `workloads` field: + +- `aiperf.sh` → `aiperf_report.csv` +- `genai_perf.sh` → `genai_perf_report.csv` +- `cmd_args.aiperf_accuracy` → `accuracy_results.csv` + +If AIPerf accuracy mode is enabled, CloudAI copies `aiperf_accuracy_artifacts/accuracy_results.csv` to `accuracy_results.csv` in the run output directory and marks the run failed if that file is not produced. + +Navigate to `./results///0/` and open the CSV to examine performance metrics. + + + +### Shared-Node Disaggregated Runs + +For Slurm, set top-level `num_nodes` lower than the sum of `prefill_worker.num-nodes` and `decode_worker.num-nodes` to run both roles on the same allocated node(s). For example, `num_nodes = 1` with `prefill_worker.num-nodes = 1` and `decode_worker.num-nodes = 1` runs one prefill worker and one decode worker on the same node. CloudAI assigns decode GPUs first and prefill GPUs after that based on each role's per-node GPU footprint, including local data-parallel ranks where applicable. The combined role GPU count must fit on one node. + +Example `aiperf_report.csv`: + + Metric,avg,min,max,p25,p50,p75,p99,std + Inter Token Latency (ms),2.81,2.66,2.88,2.79,2.83,2.84,2.87,0.04 + Time to First Token (ms),49.87,17.15,99.91,49.35,49.87,50.52,92.31,9.20 + Time to Second Token (ms),0.50,0.03,4.05,0.03,0.04,0.04,3.47,1.08 + Request Latency (ms),1652.30,1203.61,6433.87,1453.19,1462.99,1466.72,6431.16,976.18 + Output Sequence Length (tokens),498.06,410.00,501.00,500.00,500.00,500.00,501.00,12.62 + Input Sequence Length (tokens),300.00,300.00,300.00,300.00,300.00,300.00,300.00,0.00 + + Metric,Value + Output Token Throughput (tokens/sec),598.78 + Total Token Throughput (tokens/sec),962.32 + Request Throughput (requests/sec),1.20 + Request Count,50.00 + + + +## Supported Backends + +The following backends are available via the `conf/experimental/ai_dynamo/test/` directory: + +- **vLLM** (`vllm.toml`) — use with `test_scenario/vllm_slurm.toml` +- **vLLM multinode worker** — use `test_scenario/vllm_multinode_worker_slurm.toml` +- **vLLM WideEP** — use `test_scenario/vllm_wideep_slurm.toml` +- **vLLM with LMCache config propagation** — use self-contained scenario `test_scenario/vllm_lmcache.toml` +- **sglang** (`sglang.toml`) — use with `test_scenario/sglang_slurm.toml` +- **SGLang multinode worker** — use `test_scenario/sglang_multinode_worker_slurm.toml` +- **SGLang WideEP** — use `test_scenario/sglang_wideep_slurm.toml` + +Both backends use `aiperf` as the default benchmark tool and support disaggregated prefill/decode. + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/ai_dynamo.rst b/doc/workloads/ai_dynamo.rst deleted file mode 100644 index f3e474d23..000000000 --- a/doc/workloads/ai_dynamo.rst +++ /dev/null @@ -1,517 +0,0 @@ -AI Dynamo -========= - -AI Dynamo workload (`test_template_name` is ``AIDynamo``) runs AI inference benchmarks using the Dynamo framework with distributed prefill and decode workers. - - -Run Using Kubernetes --------------------- - -Prepare Cluster -~~~~~~~~~~~~~~~ -Before running the AI Dynamo workload on a Kubernetes cluster, ensure that the cluster is set up according to the instructions in the `official documentation`_. Below is a short summary of the required steps: - -.. _official documentation: https://docs.nvidia.com/dynamo/dev/getting-started/kubernetes-deployment - -.. code-block:: bash - - export NAMESPACE=dynamo-system - export RELEASE_VERSION=0.7.0 # replace with the desired release version - - helm upgrade -n default -i dynamo-crds https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-crds-${RELEASE_VERSION}.tgz - helm upgrade -n default -i dynamo-platform https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-${RELEASE_VERSION}.tgz - - # The following components are required for multi node only. - # Versions should be aligned with Dynamo version. - helm upgrade -n default -i grove oci://ghcr.io/ai-dynamo/grove/grove-charts:v0.0.0-gd462e65 - helm upgrade -n default -i kai-scheduler oci://ghcr.io/nvidia/kai-scheduler/kai-scheduler:0.0.0-4c29820 - -Launch and Monitor the Job -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. note:: - - Both CloudAI and Dynamo will try to access HuggingFace Hub. To avoid ``429 Too Many Requests`` errors and access models under auth, it is recommended to define ``HF_TOKEN`` environment variable before invoking CloudAI. Once set, run ``uv run hf auth login`` to authenticate. - -.. code-block:: bash - - uv run cloudai run --system-config \ - --tests-dir conf/experimental/ai_dynamo/test \ - --test-scenario conf/experimental/ai_dynamo/test_scenario/vllm_k8s.toml - -Run Using Slurm ---------------- - -Node Configuration for AI Dynamo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -AI Dynamo jobs use three distinct types of nodes: - -- **Frontend node**: Hosts the coordination services (`etcd`, `nats`), the **frontend server**, the **request generator** (`aiperf` by default, configurable via ``workloads`` in the test TOML), and the first decode worker -- **Prefill node(s)**: Handle the prefill stage of inference -- **Decode node(s)**: Handle the decode stage of inference (optional, depending on model and setup) - -By default, when ``num_nodes`` is omitted, CloudAI allocates separate nodes for prefill and decode workers: - -:: - - num_prefill_nodes + num_decode_nodes - -Set top-level ``num_nodes`` explicitly to control the Slurm allocation. A value lower than -``num_prefill_nodes + num_decode_nodes`` enables shared-node disaggregated inference, where prefill and decode roles -run on the same allocated node(s) with separate GPU slices. - -All node role assignments and orchestration are automatically managed by CloudAI. - -Multinode Backend Workers on Slurm -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``prefill_worker.num-nodes`` and ``decode_worker.num-nodes`` specify the total nodes assigned to each role. -Set ``nodes-per-worker`` to group those nodes into logical backend workers. For example, the following configuration -creates one prefill worker spanning two nodes and one decode worker spanning two other nodes: - -.. code-block:: toml - - [cmd_args.dynamo.prefill_worker] - num-nodes = 2 - nodes-per-worker = 2 - [cmd_args.dynamo.prefill_worker.args] - tensor-parallel-size = 8 - pipeline-parallel-size = 1 - - [cmd_args.dynamo.decode_worker] - num-nodes = 2 - nodes-per-worker = 2 - [cmd_args.dynamo.decode_worker.args] - tensor-parallel-size = 8 - pipeline-parallel-size = 1 - -When ``nodes-per-worker`` is omitted, it defaults to ``1`` and preserves the previous one-worker-per-node behavior. -For vLLM TP/PP groups, CloudAI launches rank 0 as the Dynamo-facing process and the remaining ranks with -``--headless`` using Dynamo's multiprocessing executor. When ``data-parallel-size`` is greater than one, CloudAI -instead launches a full Dynamo vLLM process on every node and derives ``data-parallel-size-local``, -``data-parallel-start-rank``, the coordinator address, and the RPC port from the worker topology. SGLang receives -``--dist-init-addr``, ``--nnodes``, and ``--node-rank`` on every participating node; its configured -``data-parallel-size`` is forwarded as ``dp-size``. The launcher derives prefill and decode disaggregation modes -from the configured roles, so worker commands do not need role flags. - -For TP/PP-spanning workers, ``tensor-parallel-size * pipeline-parallel-size`` must be divisible by -``nodes-per-worker``. For vLLM multinode data parallelism, ``data-parallel-size`` must instead be divisible by -``nodes-per-worker``, and each DP rank's TP/PP group must fit within one node. SGLang multinode data parallelism -requires ``--enable-dp-attention``. The resulting per-node GPU footprint must fit on each node. Ray-backed Dynamo -vLLM groups are not currently supported. The Slurm launcher explicitly selects etcd discovery and the NATS event -plane. - -No separate WideEP mode is required. Multinode data-parallel orchestration is inferred from -``nodes-per-worker > 1`` and ``data-parallel-size > 1``. Backend expert-parallel parameters such as vLLM's -``--enable-expert-parallel`` or SGLang's ``ep-size`` and ``--enable-dp-attention`` remain explicit backend -configuration. -CloudAI uses the allocated Slurm node names for multinode rendezvous. The cluster must make those names consistently -resolvable and mutually reachable from every allocated node and workload container. Multi-homed systems should also -configure backend network selection, such as ``GLOO_SOCKET_IFNAME`` and ``NCCL_SOCKET_IFNAME``, at the system level. -Explicit backend rendezvous addresses take precedence over addresses derived from the Slurm node names. -Clusters that cannot provide this hostname contract must supply backend-native per-node address overrides, such as -``SGLANG_HOST_IP`` or ``VLLM_NIXL_SIDE_CHANNEL_HOST``, together with explicit worker rendezvous arguments. - -Dedicated examples are available in ``test_scenario/vllm_multinode_worker_slurm.toml`` and -``test_scenario/sglang_multinode_worker_slurm.toml``. The existing ``vllm_slurm.toml`` and ``sglang_slurm.toml`` -scenarios retain their previous configurations. The vLLM example inherits ``tensor-parallel-size = 8`` from -``test/vllm.toml``; the SGLang example sets it explicitly because ``test/sglang.toml`` defaults to ``1``. -WideEP examples are provided in ``test_scenario/vllm_wideep_slurm.toml`` and -``test_scenario/sglang_wideep_slurm.toml``. - -Launch and Monitor the Job -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To run the job: - -.. code-block:: bash - - uv run cloudai run --system-config \ - --tests-dir conf/experimental/ai_dynamo/test \ - --test-scenario conf/experimental/ai_dynamo/test_scenario/vllm_slurm.toml - -The job progress monitoring can be done using either of the following options: - -.. code-block:: bash - - watch squeue --me - -.. code-block:: bash - - watch tail -n 4 ./results//*.txt - -The frontend node will initially wait to allow weight loading on all nodes. Once ready, it will launch the configured benchmark tool (``aiperf`` by default), which begins generating requests to the frontend server. All servers cooperate to complete inference, and the output will appear in ``stdout.txt``. - -Choosing a Benchmark Tool -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The benchmark tool is controlled by the ``workloads`` field in the test TOML. Set ``aiperf.sh`` to use AIPerf: - -.. code-block:: toml - - [cmd_args] - workloads = "aiperf.sh" # uses aiperf, writes aiperf_report.csv - -To use genai-perf, set: - -.. code-block:: toml - - [cmd_args] - workloads = "genai_perf.sh" # uses genai-perf, writes genai_perf_report.csv - - [cmd_args.genai_perf] - cmd = "genai-perf profile" - extra-args = "--streaming --verbose -- -v --async" - - [cmd_args.genai_perf.args] - endpoint-type = "chat" - output-tokens-mean = 500 - request-count = 50 - -AIPerf Multi-Phase Runs -~~~~~~~~~~~~~~~~~~~~~~~ - -``cmd_args.aiperf`` is the base AIPerf config. ``cmd_args.aiperf_phases`` can run several AIPerf rounds against the -same live Dynamo stack. By default, CloudAI does not restart prefill, decode, or router processes between phases: - -.. code-block:: toml - - dse_excluded_args = ["cmd_args.aiperf_phases"] - - [cmd_args.aiperf] - health-check-between-phases = true - between-phase-cmd = "true" # default no-op - - [cmd_args.aiperf.args] - request-count = 50 - server-metrics = "auto" - - [[cmd_args.aiperf_phases]] - name = "round_1" - [cmd_args.aiperf_phases.args] - concurrency = 2 - - [[cmd_args.aiperf_phases]] - name = "round_2" - [cmd_args.aiperf_phases.args] - concurrency = 4 - -Single-phase runs keep the old artifact layout: ``aiperf_artifacts/``, ``aiperf.log``, and ``aiperf_report.csv``. -Multi-phase runs write per-phase artifacts/logs/reports and copy the last phase report to ``aiperf_report.csv`` for -existing report generation. - -``between-phase-cmd`` is a bash command run after each non-final phase. The default is a no-op. Set it explicitly for -backend-specific cache cleanup, for example ``/cloudai_run_results/routerctl.sh restart`` if a test needs to restart the -Dynamo router between phases. ``health-check-between-phases`` probes the frontend after the command. - -Comparison Reports ------------------- - -CloudAI provides the scenario-level ``ai_dynamo_comparison`` report. It compares AI Dynamo runs using the standard -LLM serving tables and charts for TTFT, TPOT, successful prompts, output-token throughput, TPS per user, and TPS per -GPU. When ``aiperf_accuracy`` is configured and its result is available, the report also compares model accuracy. -Both AIPerf and GenAI-Perf CSV output are supported. - -AIPerf args are rendered as normal CLI flags. Multi-value AIPerf options should be passed with AIPerf CLI syntax, such -as ``server-metrics-formats = "csv,json,jsonl"`` or ``gpu-telemetry = "node1:9401,node2:9401"``. ``server-metrics = -"auto"`` expands to the frontend metrics endpoint, Dynamo worker metrics endpoints, and any CloudAI-started DCGM -exporters. - -Propagating LMCache Configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -AIDynamo can pass an LMCache YAML config to the worker processes by setting ``LMCACHE_CONFIG_FILE`` inside the -container. This only propagates the LMCache configuration; the vLLM/SGLang runtime still needs to be launched with the -appropriate LMCache or KV-transfer connector for that image/version. - -The preferred form is structured TOML under ``[cmd_args.lmcache]``. CloudAI converts that object to YAML in the -run output directory, mounts that directory as ``/cloudai_run_results``, and exports the generated file path as -``LMCACHE_CONFIG_FILE``: - -.. code-block:: toml - - [cmd_args] - [cmd_args.lmcache] - chunk_size = 256 - local_cpu = true - controller_pull_url = "{frontend_node}:8300" - controller_reply_url = "{frontend_node}:8400" - lmcache_worker_ports = [8788, 8789, 8790, 8791] - max_local_cpu_size = 6.0 - nixl_buffer_size = 2079377920 - nixl_buffer_device = "cpu" - - [cmd_args.lmcache.extra_config] - enable_nixl_storage = false - nixl_backend = "POSIX" - nixl_path = "{storage_cache_dir}" - nixl_pool_size = 2048 - -For an example that uses test-in-scenario mode, see -``conf/experimental/ai_dynamo/test_scenario/vllm_lmcache.toml``. Because the test is fully defined inside the scenario, -``--tests-dir`` is not required when running that example: - -.. code-block:: bash - - uv run cloudai run --system-config \ - --test-scenario conf/experimental/ai_dynamo/test_scenario/vllm_lmcache.toml - -The example sets ``dse_excluded_args = ["cmd_args.lmcache.lmcache_worker_ports"]`` because -``lmcache_worker_ports`` is a list-valued LMCache setting, not a DSE sweep dimension. Other list-valued LMCache fields -can still be swept unless their ``cmd_args.`` path is also excluded. - -Alternatively, mount your own LMCache YAML file with ``extra_container_mounts`` and set ``LMCACHE_CONFIG_FILE`` through -``extra_env_vars``: - -.. code-block:: toml - - extra_container_mounts = ["/host/lmcache:/lmcache"] - extra_env_vars = { LMCACHE_CONFIG_FILE = "/lmcache/config.yaml" } - -For multi-node LMCache storage tests, any path referenced by the LMCache YAML, such as ``nixl_path`` for POSIX-backed -storage, must be visible and writable from every node that is expected to share cached data. A node-local path such as -``/tmp`` is suitable only for single-node smoke tests or configuration propagation checks. - -LMCache YAML values can use runtime placeholders. CloudAI renders them inside the Slurm job before launching workers: -``{frontend_node}``, ``{frontend_ip}``, ``{results_dir}``, and ``{storage_cache_dir}``. Unknown placeholders fail the -run before worker processes start. - -Propagating HiCache Configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Define a HiCache storage-backend configuration under ``[cmd_args.hicache]``. CloudAI serializes the object as -``hicache-config.toml`` in the run output directory, exposes its container path through ``HICACHE_CONFIG_FILE``, and -passes the resolved path to both workers as ``--hicache-storage-backend-extra-config``: - -.. code-block:: toml - - [cmd_args] - [cmd_args.hicache.plugin.posix] - active = true - use_uring = "false" - - [cmd_args.dynamo.prefill_worker] - extra-args = "--enable-hierarchical-cache" - - [cmd_args.dynamo.prefill_worker.args] - hicache-size = 8 - hicache-ratio = 0 - hicache-write-policy = "write_through" - hicache-mem-layout = "page_first" - hicache-storage-prefetch-policy = "wait_complete" - hicache-storage-backend = "nixl" - - [cmd_args.dynamo.decode_worker] - extra-args = "--enable-hierarchical-cache" - - [cmd_args.dynamo.decode_worker.args] - hicache-size = 8 - hicache-ratio = 0 - hicache-write-policy = "write_through" - hicache-mem-layout = "page_first" - hicache-storage-prefetch-policy = "wait_complete" - hicache-storage-backend = "nixl" - -The remaining HiCache engine options stay in the worker ``args`` sections so they can be configured or swept through -DSE normally. - -The runtime replacement mechanism below supports per-node HiCache values. For example, change the POSIX setting above -to ``use_uring = "${use_uring}"`` and have the startup manifest replace ``${use_uring}`` with ``false`` or ``true``. -The worker receives the resulting per-node TOML path automatically. - -For a self-contained single-node smoke test using NIXL's POSIX plugin, run the following on a node with at least two -GPUs (one each for the prefill and decode workers): - -.. code-block:: bash - - uv run cloudai run --system-config \ - --test-scenario conf/experimental/ai_dynamo/test_scenario/sglang_hicache.toml - -Per-node Startup Commands and Runtime Replacements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -On Slurm, ``startup_cmd`` runs once per allocated node before the workload. It runs on the host by default. Set -``startup_cmd_docker_image`` to run it in a container using the workload's mounts: - -.. code-block:: toml - - [cmd_args] - startup_cmd = "/mnt/discovery/discover-device.sh" - startup_cmd_docker_image = "nvcr.io/example/discovery-tools:latest" - -CloudAI exports ``CLOUDAI_RUNTIME_FILE`` to the command. It points to ``runtime/.json`` in the test output -directory, or ``/cloudai_run_results/runtime/.json`` inside a startup container. Write replacements there: - -.. code-block:: json - - { - "LMCACHE_CONFIG_FILE": { - "${device}": "/dev/ng4n1", - "${namespace_id}": "1" - }, - "HICACHE_CONFIG_FILE": { - "${use_uring}": "false" - } - } - -Each top-level key names an environment variable containing a config path. AIDynamo creates a per-node copy, applies -literal replacements, then updates the variable to point to that copy. - -For ``lmcache-config.yaml`` on ``node-01``, the result is ``lmcache-config.node-01.yaml``. The shared source remains -unchanged. LMCache placeholders are rendered afterward in the per-node copy. Missing manifest means no replacements. - -Startup runs once per test case in normal and ``--single-sbatch`` modes. - -If the selected LMCache mode needs a controller, CloudAI can start one on the frontend node: - -.. code-block:: toml - - [cmd_args.lmcache_controller] - cmd = "lmcache_controller --host 0.0.0.0 --port 9000 --monitor-ports {\"pull\":8300,\"reply\":8400}" - -This only launches the process. For disaggregated or multi-node runs, the LMCache YAML still needs controller addresses -that resolve to the frontend node from every worker. With the default controller monitor ports, use -``controller_pull_url = "{frontend_node}:8300"`` and ``controller_reply_url = "{frontend_node}:8400"``. The -``lmcache_worker_ports`` list must match the number of worker ranks. - -Semantic Degradation With AIPerf Accuracy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -AIDynamo uses AIPerf accuracy mode as its semantic degradation signal. Enable it with -``[cmd_args.aiperf_accuracy]``. This runs after the configured performance workload, so it can be used with either -``aiperf.sh`` or ``genai_perf.sh``: - -.. code-block:: toml - - [cmd_args] - workloads = "aiperf.sh" - - [cmd_args.aiperf] - [cmd_args.aiperf.args] - request-count = 50 - synthetic-input-tokens-mean = 300 - output-tokens-mean = 500 - concurrency = 2 - - [cmd_args.aiperf_accuracy] - entrypoint = "aiperf profile" - setup-cmd = "python -m pip install --break-system-packages --upgrade aiperf==0.8.0" - cli = ''' - --model {model} - --url {url} - --endpoint-type chat - --streaming - --artifact-dir {artifact_dir} - --no-server-metrics - --accuracy-benchmark mmlu - --accuracy-n-shots 5 - --accuracy-tasks abstract_algebra - --concurrency 10 - --extra-inputs '{"temperature":0,"chat_template_kwargs":{"enable_thinking":false}}' - --num-requests 100 - ''' - -When ``cmd_args.aiperf_accuracy`` is configured, CloudAI expects AIPerf to produce ``accuracy_results.csv`` and exposes -the ``accuracy`` metric from its ``OVERALL`` row. The metric is reported as a 0.0-1.0 fraction. Keep synthetic prompt -and token-length flags out of this mode; the benchmark dataset should come from AIPerf's accuracy benchmark. - -The ``entrypoint`` and ``cli`` fields form the accuracy command. CloudAI expands ``{model}``, ``{url}``, -``{endpoint}``, ``{result_dir}``, and ``{artifact_dir}`` in ``cli`` before launching it. The ``setup-cmd`` field is -optional. It is useful for Dynamo images that include an older system ``aiperf`` build without the accuracy benchmark -plugins. The example upgrades the image-level ``aiperf`` before launching ``aiperf profile``. -MMLU is loaded from ``lighteval/mmlu``, so either allow Hugging Face dataset access or pre-cache that dataset before -running with ``HF_HUB_OFFLINE``/``HF_DATASETS_OFFLINE`` enabled. -For Qwen3 models, the example disables thinking mode so short MMLU answers can be parsed as choices. - -Custom Accuracy Scripts -~~~~~~~~~~~~~~~~~~~~~~~ - -``cmd_args.aiperf_accuracy`` can also launch a custom mounted script instead of AIPerf. Mount the script or its parent -directory with ``extra_container_mounts`` and set ``entrypoint`` to the in-container command: - -.. code-block:: toml - - extra_container_mounts = ["/host/custom_accuracy:/custom_accuracy"] - - [cmd_args.aiperf_accuracy] - entrypoint = "python /custom_accuracy/dummy_accuracy.py" - cli = "--model {model} --url {url} --endpoint {endpoint} --artifact-dir {artifact_dir} --prompt ping" - -CloudAI expands placeholders in ``cli`` and runs ``entrypoint`` with that CLI string. The custom command must write -``accuracy_results.csv`` inside ``{artifact_dir}`` with an ``OVERALL`` row. CloudAI copies that file to the run output -directory and exposes the same ``accuracy`` metric as AIPerf accuracy mode. - -Review Benchmark Results ------------------------- - -After job completion, CloudAI places output logs and result files in the designated results directory. The result file name depends on the configured ``workloads`` field: - -- ``aiperf.sh`` → ``aiperf_report.csv`` -- ``genai_perf.sh`` → ``genai_perf_report.csv`` -- ``cmd_args.aiperf_accuracy`` → ``accuracy_results.csv`` - -If AIPerf accuracy mode is enabled, CloudAI copies ``aiperf_accuracy_artifacts/accuracy_results.csv`` to -``accuracy_results.csv`` in the run output directory and marks the run failed if that file is not produced. - -Navigate to ``./results///0/`` and open the CSV to examine performance metrics. - -Shared-Node Disaggregated Runs -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For Slurm, set top-level ``num_nodes`` lower than the sum of ``prefill_worker.num-nodes`` and -``decode_worker.num-nodes`` to run both roles on the same allocated node(s). For example, ``num_nodes = 1`` with -``prefill_worker.num-nodes = 1`` and ``decode_worker.num-nodes = 1`` runs one prefill worker and one decode worker on -the same node. CloudAI assigns decode GPUs first and prefill GPUs after that based on each role's per-node GPU -footprint, including local data-parallel ranks where applicable. The combined role GPU count must fit on one node. - -Example ``aiperf_report.csv``: - -:: - - Metric,avg,min,max,p25,p50,p75,p99,std - Inter Token Latency (ms),2.81,2.66,2.88,2.79,2.83,2.84,2.87,0.04 - Time to First Token (ms),49.87,17.15,99.91,49.35,49.87,50.52,92.31,9.20 - Time to Second Token (ms),0.50,0.03,4.05,0.03,0.04,0.04,3.47,1.08 - Request Latency (ms),1652.30,1203.61,6433.87,1453.19,1462.99,1466.72,6431.16,976.18 - Output Sequence Length (tokens),498.06,410.00,501.00,500.00,500.00,500.00,501.00,12.62 - Input Sequence Length (tokens),300.00,300.00,300.00,300.00,300.00,300.00,300.00,0.00 - - Metric,Value - Output Token Throughput (tokens/sec),598.78 - Total Token Throughput (tokens/sec),962.32 - Request Throughput (requests/sec),1.20 - Request Count,50.00 - -Supported Backends ------------------- - -The following backends are available via the ``conf/experimental/ai_dynamo/test/`` directory: - -- **vLLM** (``vllm.toml``) — use with ``test_scenario/vllm_slurm.toml`` -- **vLLM multinode worker** — use ``test_scenario/vllm_multinode_worker_slurm.toml`` -- **vLLM WideEP** — use ``test_scenario/vllm_wideep_slurm.toml`` -- **vLLM with LMCache config propagation** — use self-contained scenario ``test_scenario/vllm_lmcache.toml`` -- **sglang** (``sglang.toml``) — use with ``test_scenario/sglang_slurm.toml`` -- **SGLang multinode worker** — use ``test_scenario/sglang_multinode_worker_slurm.toml`` -- **SGLang WideEP** — use ``test_scenario/sglang_wideep_slurm.toml`` - -Both backends use ``aiperf`` as the default benchmark tool and support disaggregated prefill/decode. - - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/aiconfigurator.mdx b/doc/workloads/aiconfigurator.mdx new file mode 100644 index 000000000..311834bf8 --- /dev/null +++ b/doc/workloads/aiconfigurator.mdx @@ -0,0 +1,110 @@ + + +AIConfigurator workload (`test_template_name` is `Aiconfigurator`) runs the AIConfigurator predictor using the installed `aiconfigurator` Python package. It is a **Standalone** workload (no Slurm/Kubernetes/RunAI required). + + + +## Outputs + +Each test run produces: + +- `report.json`: Predictor output (JSON dict of metrics and metadata) +- `stdout.txt` / `stderr.txt`: Predictor logs +- `run_simple_predictor.sh`: Repro script containing the exact executed command (useful for debugging) + + + +## Usage Example + +Test TOML example (Disaggregated mode): + +``` toml +name = "aiconfigurator_disagg_demo" +description = "Example AIConfigurator disaggregated predictor" +test_template_name = "Aiconfigurator" + +[cmd_args] +model_name = "LLAMA3.1_70B" +system = "h200_sxm" +backend = "trtllm" +version = "0.20.0" +isl = 4000 +osl = 500 + + [cmd_args.disagg] + p_tp = 1 + p_pp = 1 + p_dp = 1 + p_bs = 1 + p_workers = 1 + + d_tp = 1 + d_pp = 1 + d_dp = 1 + d_bs = 8 + d_workers = 2 + + prefill_correction_scale = 1.0 + decode_correction_scale = 1.0 +``` + +Test TOML example (Aggregated/IFB mode): + +``` toml +name = "aiconfigurator_agg_demo" +description = "Example AIConfigurator aggregated predictor" +test_template_name = "Aiconfigurator" + +[cmd_args] +model_name = "LLAMA3.1_70B" +system = "h200_sxm" +backend = "trtllm" +version = "0.20.0" +isl = 4000 +osl = 500 + + [cmd_args.agg] + batch_size = 8 + ctx_tokens = 16 + tp = 1 + pp = 1 + dp = 1 +``` + + + +## Running + +``` bash +uv run cloudai run --system-config conf/common/system/standalone_system.toml \ + --tests-dir conf/experimental/aiconfigurator/test \ + --test-scenario conf/experimental/aiconfigurator/test_scenario/aiconfigurator_disagg.toml +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + + + + +### Command Generation Strategy (Standalone) + + + + + +### Report Generation Strategy + + diff --git a/doc/workloads/aiconfigurator.rst b/doc/workloads/aiconfigurator.rst deleted file mode 100644 index 64a8a231c..000000000 --- a/doc/workloads/aiconfigurator.rst +++ /dev/null @@ -1,112 +0,0 @@ -AIConfigurator -============== - -AIConfigurator workload (`test_template_name` is ``Aiconfigurator``) runs the AIConfigurator predictor using the installed -``aiconfigurator`` Python package. It is a **Standalone** workload (no Slurm/Kubernetes/RunAI required). - -Outputs -------- - -Each test run produces: - -- ``report.json``: Predictor output (JSON dict of metrics and metadata) -- ``stdout.txt`` / ``stderr.txt``: Predictor logs -- ``run_simple_predictor.sh``: Repro script containing the exact executed command (useful for debugging) - -Usage Example -------------- - -Test TOML example (Disaggregated mode): - -.. code-block:: toml - - name = "aiconfigurator_disagg_demo" - description = "Example AIConfigurator disaggregated predictor" - test_template_name = "Aiconfigurator" - - [cmd_args] - model_name = "LLAMA3.1_70B" - system = "h200_sxm" - backend = "trtllm" - version = "0.20.0" - isl = 4000 - osl = 500 - - [cmd_args.disagg] - p_tp = 1 - p_pp = 1 - p_dp = 1 - p_bs = 1 - p_workers = 1 - - d_tp = 1 - d_pp = 1 - d_dp = 1 - d_bs = 8 - d_workers = 2 - - prefill_correction_scale = 1.0 - decode_correction_scale = 1.0 - -Test TOML example (Aggregated/IFB mode): - -.. code-block:: toml - - name = "aiconfigurator_agg_demo" - description = "Example AIConfigurator aggregated predictor" - test_template_name = "Aiconfigurator" - - [cmd_args] - model_name = "LLAMA3.1_70B" - system = "h200_sxm" - backend = "trtllm" - version = "0.20.0" - isl = 4000 - osl = 500 - - [cmd_args.agg] - batch_size = 8 - ctx_tokens = 16 - tp = 1 - pp = 1 - dp = 1 - -Running -------- - -.. code-block:: bash - - uv run cloudai run --system-config conf/common/system/standalone_system.toml \ - --tests-dir conf/experimental/aiconfigurator/test \ - --test-scenario conf/experimental/aiconfigurator/test_scenario/aiconfigurator_disagg.toml - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorTestDefinition - :members: - :show-inheritance: - -Command Generation Strategy (Standalone) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.aiconfig.standalone_command_gen_strategy.AiconfiguratorStandaloneCommandGenStrategy - :members: - :show-inheritance: - -Report Generation Strategy -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.aiconfig.report_generation_strategy.AiconfiguratorReportGenerationStrategy - :members: - :show-inheritance: diff --git a/doc/workloads/bash_cmd.mdx b/doc/workloads/bash_cmd.mdx new file mode 100644 index 000000000..6622249ef --- /dev/null +++ b/doc/workloads/bash_cmd.mdx @@ -0,0 +1,67 @@ + + +Bash Command workload (`test_template_name` is `BashCmd`) allows users to execute arbitrary bash commands within the CloudAI framework. This is useful for simple scripts, custom testing commands, or integrating external tools. + +`cmd` specified in the `cmd_args` section will be added as-is into generated sbatch script. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_bash_test" +description = "Example bash command test" +test_template_name = "BashCmd" + +[cmd_args] +cmd = "echo 'Hello from CloudAI!'" +``` + +Test Scenario example: + +``` toml +name = "bash-test" + +[[Tests]] +id = "bash.1" +num_nodes = 1 +time_limit = "00:05:00" + +test_name = "my_bash_test" +``` + +Test-in-Scenario example: + +``` toml +name = "bash-test" + +[[Tests]] +id = "bash.1" +num_nodes = 1 +time_limit = "00:05:00" + +name = "my_bash_test" +description = "Example bash command test" +test_template_name = "BashCmd" + + [Tests.cmd_args] + cmd = "echo 'Hello from CloudAI!'" +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/bash_cmd.rst b/doc/workloads/bash_cmd.rst deleted file mode 100644 index 06d4fa0b2..000000000 --- a/doc/workloads/bash_cmd.rst +++ /dev/null @@ -1,68 +0,0 @@ -Bash Command -============ - -Bash Command workload (`test_template_name` is ``BashCmd``) allows users to execute arbitrary bash commands within the CloudAI framework. This is useful for simple scripts, custom testing commands, or integrating external tools. - -``cmd`` specified in the ``cmd_args`` section will be added as-is into generated sbatch script. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_bash_test" - description = "Example bash command test" - test_template_name = "BashCmd" - - [cmd_args] - cmd = "echo 'Hello from CloudAI!'" - -Test Scenario example: - -.. code-block:: toml - - name = "bash-test" - - [[Tests]] - id = "bash.1" - num_nodes = 1 - time_limit = "00:05:00" - - test_name = "my_bash_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "bash-test" - - [[Tests]] - id = "bash.1" - num_nodes = 1 - time_limit = "00:05:00" - - name = "my_bash_test" - description = "Example bash command test" - test_template_name = "BashCmd" - - [Tests.cmd_args] - cmd = "echo 'Hello from CloudAI!'" - -API Documentation ---------------------------------- - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.bash_cmd.bash_cmd.BashCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.bash_cmd.bash_cmd.BashCmdTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/chakra_replay.mdx b/doc/workloads/chakra_replay.mdx new file mode 100644 index 000000000..d6531460e --- /dev/null +++ b/doc/workloads/chakra_replay.mdx @@ -0,0 +1,65 @@ + + +Chakra Replay workload (`test_template_name` is `ChakraReplay`) replays execution traces from the Chakra execution trace format for performance analysis and debugging. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_chakra_test" +description = "Example Chakra replay test" +test_template_name = "ChakraReplay" + +[cmd_args] +trace_path = "/path/to/trace.et" +``` + +Test Scenario example: + +``` toml +name = "chakra-replay-test" + +[[Tests]] +id = "chakra.1" +num_nodes = 1 +time_limit = "00:10:00" + +test_name = "my_chakra_test" +``` + +Test-in-Scenario example: + +``` toml +name = "chakra-replay-test" + +[[Tests]] +id = "chakra.1" +num_nodes = 1 +time_limit = "00:10:00" + +name = "my_chakra_test" +description = "Example Chakra replay test" +test_template_name = "ChakraReplay" + + [Tests.cmd_args] + trace_path = "/path/to/trace.et" +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/chakra_replay.rst b/doc/workloads/chakra_replay.rst deleted file mode 100644 index 663fd62b5..000000000 --- a/doc/workloads/chakra_replay.rst +++ /dev/null @@ -1,66 +0,0 @@ -Chakra Replay -============= - -Chakra Replay workload (`test_template_name` is ``ChakraReplay``) replays execution traces from the Chakra execution trace format for performance analysis and debugging. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_chakra_test" - description = "Example Chakra replay test" - test_template_name = "ChakraReplay" - - [cmd_args] - trace_path = "/path/to/trace.et" - -Test Scenario example: - -.. code-block:: toml - - name = "chakra-replay-test" - - [[Tests]] - id = "chakra.1" - num_nodes = 1 - time_limit = "00:10:00" - - test_name = "my_chakra_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "chakra-replay-test" - - [[Tests]] - id = "chakra.1" - num_nodes = 1 - time_limit = "00:10:00" - - name = "my_chakra_test" - description = "Example Chakra replay test" - test_template_name = "ChakraReplay" - - [Tests.cmd_args] - trace_path = "/path/to/trace.et" - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/ddlb.mdx b/doc/workloads/ddlb.mdx new file mode 100644 index 000000000..99d4575e4 --- /dev/null +++ b/doc/workloads/ddlb.mdx @@ -0,0 +1,75 @@ + + +DDLB workload (`test_template_name` is `DDLB`) allows users to execute DDLB (Distributed Deep Learning Benchmarks) within the CloudAI framework. Please find the DDLB README at [https://github.com/samnordmann/ddlb](https://github.com/samnordmann/ddlb). + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_ddlb_test" +description = "Example DDLB test" +test_template_name = "DDLB" + +[cmd_args] +docker_image_url = "" +primitive = "tp_columnwise" +dtype = "float16" +``` + +Test Scenario example: + +``` toml +name = "ddlb-test" + +[[Tests]] +id = "ddlb.1" +num_nodes = 1 +time_limit = "00:10:00" + +test_name = "my_ddlb_test" +``` + +Test-in-Scenario example: + +``` toml +name = "ddlb-test" + +[[Tests]] +id = "ddlb.1" +num_nodes = 1 +time_limit = "00:10:00" + +name = "my_ddlb_test" +description = "Example DDLB test" +test_template_name = "DDLB" + + [Tests.cmd_args] + docker_image_url = "" + primitive = "tp_columnwise" + m = 1024 + n = 128 + k = 1024 + dtype = "float16" + num_iterations = 50 + num_warmups = 5 + impl = "pytorch;backend=nccl;order=AG_before" +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/ddlb.rst b/doc/workloads/ddlb.rst deleted file mode 100644 index 8475ca401..000000000 --- a/doc/workloads/ddlb.rst +++ /dev/null @@ -1,76 +0,0 @@ -DDLB -==== - -DDLB workload (`test_template_name` is ``DDLB``) allows users to execute DDLB (Distributed Deep Learning Benchmarks) within the CloudAI framework. Please find the DDLB README at https://github.com/samnordmann/ddlb. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_ddlb_test" - description = "Example DDLB test" - test_template_name = "DDLB" - - [cmd_args] - docker_image_url = "" - primitive = "tp_columnwise" - dtype = "float16" - -Test Scenario example: - -.. code-block:: toml - - name = "ddlb-test" - - [[Tests]] - id = "ddlb.1" - num_nodes = 1 - time_limit = "00:10:00" - - test_name = "my_ddlb_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "ddlb-test" - - [[Tests]] - id = "ddlb.1" - num_nodes = 1 - time_limit = "00:10:00" - - name = "my_ddlb_test" - description = "Example DDLB test" - test_template_name = "DDLB" - - [Tests.cmd_args] - docker_image_url = "" - primitive = "tp_columnwise" - m = 1024 - n = 128 - k = 1024 - dtype = "float16" - num_iterations = 50 - num_warmups = 5 - impl = "pytorch;backend=nccl;order=AG_before" - -API Documentation ---------------------------------- - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.ddlb.ddlb.DDLBCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.ddlb.ddlb.DDLBTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/deepep.mdx b/doc/workloads/deepep.mdx new file mode 100644 index 000000000..4c430cbe4 --- /dev/null +++ b/doc/workloads/deepep.mdx @@ -0,0 +1,107 @@ + + +DeepEP Benchmark workload (`test_template_name` is `DeepEP`) allows users to execute DeepEP (Deep Expert Parallelism) MoE (Mixture of Experts) benchmarks within the CloudAI framework. + + + +## Overview + +DeepEP is a benchmark for measuring the performance of MoE models with distributed expert parallelism. It supports: + +- **Two operation modes**: Standard and Low-Latency +- **Multiple data types**: bfloat16 and FP8 +- **Flexible network configurations**: With or without NVLink +- **Configurable model parameters**: Experts, tokens, hidden size, top-k +- **Performance profiling**: Kineto profiler support + + + +## Usage Example + +Test TOML example (Standard Mode): + +``` toml +name = "deepep_standard" +description = "DeepEP MoE Benchmark - Standard Mode" +test_template_name = "DeepEP" + +[cmd_args] +docker_image_url = "" +mode = "standard" +tokens = 1024 +num_experts = 256 +num_topk = 8 +hidden_size = 7168 +data_type = "bfloat16" +num_warmups = 20 +num_iterations = 50 +``` + +Test TOML example (Low-Latency Mode): + +``` toml +name = "deepep_low_latency" +description = "DeepEP MoE Benchmark - Low Latency Mode" +test_template_name = "DeepEP" + +[cmd_args] +docker_image_url = "" +mode = "low_latency" +tokens = 128 +num_experts = 256 +num_topk = 1 +hidden_size = 7168 +data_type = "bfloat16" +allow_nvlink_for_low_latency = false +allow_mnnvl = false +``` + +Test Scenario example: + +``` toml +name = "deepep-benchmark" + +[[Tests]] +id = "Tests.1" +test_name = "deepep_standard" +num_nodes = 2 +time_limit = "00:30:00" +``` + +Test-in-Scenario example: + +``` toml +name = "deepep-benchmark" + +[[Tests]] +id = "Tests.1" +num_nodes = 2 +time_limit = "00:30:00" + +name = "deepep_standard" +description = "DeepEP MoE Benchmark" +test_template_name = "DeepEP" + + [Tests.cmd_args] + docker_image_url = "" + mode = "standard" + tokens = 1024 + num_experts = 256 + num_topk = 8 +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/deepep.rst b/doc/workloads/deepep.rst deleted file mode 100644 index 4c309824d..000000000 --- a/doc/workloads/deepep.rst +++ /dev/null @@ -1,107 +0,0 @@ -DeepEP Benchmark -================ - -DeepEP Benchmark workload (``test_template_name`` is ``DeepEP``) allows users to execute DeepEP (Deep Expert Parallelism) MoE (Mixture of Experts) benchmarks within the CloudAI framework. - -Overview --------- - -DeepEP is a benchmark for measuring the performance of MoE models with distributed expert parallelism. It supports: - -- **Two operation modes**: Standard and Low-Latency -- **Multiple data types**: bfloat16 and FP8 -- **Flexible network configurations**: With or without NVLink -- **Configurable model parameters**: Experts, tokens, hidden size, top-k -- **Performance profiling**: Kineto profiler support - -Usage Example -------------- - -Test TOML example (Standard Mode): - -.. code-block:: toml - - name = "deepep_standard" - description = "DeepEP MoE Benchmark - Standard Mode" - test_template_name = "DeepEP" - - [cmd_args] - docker_image_url = "" - mode = "standard" - tokens = 1024 - num_experts = 256 - num_topk = 8 - hidden_size = 7168 - data_type = "bfloat16" - num_warmups = 20 - num_iterations = 50 - -Test TOML example (Low-Latency Mode): - -.. code-block:: toml - - name = "deepep_low_latency" - description = "DeepEP MoE Benchmark - Low Latency Mode" - test_template_name = "DeepEP" - - [cmd_args] - docker_image_url = "" - mode = "low_latency" - tokens = 128 - num_experts = 256 - num_topk = 1 - hidden_size = 7168 - data_type = "bfloat16" - allow_nvlink_for_low_latency = false - allow_mnnvl = false - -Test Scenario example: - -.. code-block:: toml - - name = "deepep-benchmark" - - [[Tests]] - id = "Tests.1" - test_name = "deepep_standard" - num_nodes = 2 - time_limit = "00:30:00" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "deepep-benchmark" - - [[Tests]] - id = "Tests.1" - num_nodes = 2 - time_limit = "00:30:00" - - name = "deepep_standard" - description = "DeepEP MoE Benchmark" - test_template_name = "DeepEP" - - [Tests.cmd_args] - docker_image_url = "" - mode = "standard" - tokens = 1024 - num_experts = 256 - num_topk = 8 - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.deepep.deepep.DeepEPCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.deepep.deepep.DeepEPTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/dynamo_mocker.mdx b/doc/workloads/dynamo_mocker.mdx new file mode 100644 index 000000000..aaefe1b9e --- /dev/null +++ b/doc/workloads/dynamo_mocker.mdx @@ -0,0 +1,117 @@ + + +DynamoMocker workload (`test_template_name` is `DynamoMocker`) runs GPU-free LLM inference simulation using `dynamo.mocker` and `dynamo.frontend` from the [ai-dynamo](https://github.com/ai-dynamo/dynamo) package, then benchmarks the stack with [aiperf](https://github.com/NVIDIA/aiperf) or [genai-perf](https://github.com/triton-inference-server/client/tree/main/src/c%2B%2B/perf_analyzer/genai-perf). + +It is a **Standalone** workload (no Slurm/Kubernetes/RunAI required). + + + +## Prerequisites + +CloudAI automatically installs `ai-dynamo`, `aiperf`, and `genai-perf` into a managed Python virtual environment on first run — no manual pip install is needed. + +The one prerequisite is `nats-server`. On many clusters `nats-server` is pre-installed by administrators and is already on `PATH`. Check if it is already available: + +``` bash +which nats-server +``` + +If not found, download the binary from the [official releases](https://github.com/nats-io/nats-server/releases), extract it, and add it to `PATH`: + +``` bash +# replace with the latest release tag, e.g. v2.10.24 +curl -L https://github.com/nats-io/nats-server/releases/download//nats-server--linux-amd64.zip \ + -o /tmp/nats-server.zip +unzip /tmp/nats-server.zip -d /tmp/ +mkdir -p ~/.local/bin && mv /tmp/nats-server--linux-amd64/nats-server ~/.local/bin/ +export PATH="$HOME/.local/bin:$PATH" # add to ~/.bashrc to persist +``` + +An `HF_TOKEN` environment variable is required to download gated models from HuggingFace Hub. Set it before running: + +``` bash +export HF_TOKEN= +``` + + + +## Topologies + +The workload supports two disaggregation modes, configured via `cmd_args.worker.disaggregation_mode`: + +- **Combined** (`none`): a single `dynamo.mocker` process handles both prefill and decode. Controlled by `cmd_args.worker.num_workers`. +- **Disaggregated** (`prefill_decode`): separate prefill and decode mocker instances, mirroring the production `ai_dynamo` topology. Instance counts are set via `cmd_args.worker.prefill_worker.num_nodes` and `cmd_args.worker.decode_worker.num_nodes`. + + + +## Benchmark Tools + +Select the benchmark tool with `cmd_args.benchmark_tool`: + +- `"aiperf"` (default in the provided TOML) — uses the `aiperf` profiler +- `"genai_perf"` — uses `genai-perf profile` + +Parameters for the active tool are configured under `[cmd_args.aiperf]` or `[cmd_args.genai_perf]`. + + + +## Run Using Standalone + + + +Set `HF_TOKEN` before running to allow model download from HuggingFace Hub. + + + +``` bash +uv run cloudai run \ + --system-config conf/experimental/dynamo_mocker/system/standalone_system.toml \ + --tests-dir conf/experimental/dynamo_mocker/test \ + --test-scenario conf/experimental/dynamo_mocker/test_scenario/dynamo_mocker.toml +``` + +CloudAI will: + +1. Install `ai-dynamo`, `aiperf`, and `genai-perf` into a managed venv (first run only). +2. Write a wrapper script and launch `dynamo_mocker.sh`. +3. Start `nats-server`, `dynamo.mocker` (prefill and decode), and `dynamo.frontend`. +4. Run the benchmark and write results to the output directory. + + + +## Review Benchmark Results + +After the run completes, results are placed in `results///`: + +- `benchmark_report.csv` — full per-request and aggregate metrics (throughput, latency percentiles, TTFT, ITL) +- `stdout.txt` / `stderr.txt` — orchestration log and process output +- `dynamo_prefill_0.log`, `dynamo_decode_0.log`, `dynamo_frontend.log` — per-component logs +- `nats.log` — NATS server log + +Key summary metrics from `benchmark_report.csv`: + + Metric,Value + Output Token Throughput (tokens/sec),667.58 + Request Count,50.00 + Request Throughput (requests/sec),18.04 + + Metric,avg,p50,p99 + Request Latency (ms),507.49,475.50,893.26 + Time to First Token (ms),77.82,71.99,137.26 + Inter Token Latency (ms),12.03,11.55,16.42 + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/dynamo_mocker.rst b/doc/workloads/dynamo_mocker.rst deleted file mode 100644 index 39c2abf2c..000000000 --- a/doc/workloads/dynamo_mocker.rst +++ /dev/null @@ -1,125 +0,0 @@ -DynamoMocker -============ - -DynamoMocker workload (``test_template_name`` is ``DynamoMocker``) runs GPU-free LLM inference simulation -using ``dynamo.mocker`` and ``dynamo.frontend`` from the `ai-dynamo `_ -package, then benchmarks the stack with `aiperf `_ or -`genai-perf `_. - -It is a **Standalone** workload (no Slurm/Kubernetes/RunAI required). - -Prerequisites -------------- - -CloudAI automatically installs ``ai-dynamo``, ``aiperf``, and ``genai-perf`` into a managed Python virtual -environment on first run — no manual pip install is needed. - -The one prerequisite is ``nats-server``. On many clusters ``nats-server`` is pre-installed by administrators -and is already on ``PATH``. Check if it is already available: - -.. code-block:: bash - - which nats-server - -If not found, download the binary from the `official releases `_, -extract it, and add it to ``PATH``: - -.. code-block:: bash - - # replace with the latest release tag, e.g. v2.10.24 - curl -L https://github.com/nats-io/nats-server/releases/download//nats-server--linux-amd64.zip \ - -o /tmp/nats-server.zip - unzip /tmp/nats-server.zip -d /tmp/ - mkdir -p ~/.local/bin && mv /tmp/nats-server--linux-amd64/nats-server ~/.local/bin/ - export PATH="$HOME/.local/bin:$PATH" # add to ~/.bashrc to persist - - -An ``HF_TOKEN`` environment variable is required to download gated models from HuggingFace Hub. Set it before -running: - -.. code-block:: bash - - export HF_TOKEN= - -Topologies ----------- - -The workload supports two disaggregation modes, configured via ``cmd_args.worker.disaggregation_mode``: - -- **Combined** (``none``): a single ``dynamo.mocker`` process handles both prefill and decode. Controlled - by ``cmd_args.worker.num_workers``. -- **Disaggregated** (``prefill_decode``): separate prefill and decode mocker instances, mirroring the - production ``ai_dynamo`` topology. Instance counts are set via - ``cmd_args.worker.prefill_worker.num_nodes`` and ``cmd_args.worker.decode_worker.num_nodes``. - -Benchmark Tools ---------------- - -Select the benchmark tool with ``cmd_args.benchmark_tool``: - -- ``"aiperf"`` (default in the provided TOML) — uses the ``aiperf`` profiler -- ``"genai_perf"`` — uses ``genai-perf profile`` - -Parameters for the active tool are configured under ``[cmd_args.aiperf]`` or ``[cmd_args.genai_perf]``. - -Run Using Standalone --------------------- - -.. note:: - - Set ``HF_TOKEN`` before running to allow model download from HuggingFace Hub. - -.. code-block:: bash - - uv run cloudai run \ - --system-config conf/experimental/dynamo_mocker/system/standalone_system.toml \ - --tests-dir conf/experimental/dynamo_mocker/test \ - --test-scenario conf/experimental/dynamo_mocker/test_scenario/dynamo_mocker.toml - -CloudAI will: - -1. Install ``ai-dynamo``, ``aiperf``, and ``genai-perf`` into a managed venv (first run only). -2. Write a wrapper script and launch ``dynamo_mocker.sh``. -3. Start ``nats-server``, ``dynamo.mocker`` (prefill and decode), and ``dynamo.frontend``. -4. Run the benchmark and write results to the output directory. - -Review Benchmark Results ------------------------- - -After the run completes, results are placed in ``results///``: - -- ``benchmark_report.csv`` — full per-request and aggregate metrics (throughput, latency percentiles, TTFT, ITL) -- ``stdout.txt`` / ``stderr.txt`` — orchestration log and process output -- ``dynamo_prefill_0.log``, ``dynamo_decode_0.log``, ``dynamo_frontend.log`` — per-component logs -- ``nats.log`` — NATS server log - -Key summary metrics from ``benchmark_report.csv``: - -:: - - Metric,Value - Output Token Throughput (tokens/sec),667.58 - Request Count,50.00 - Request Throughput (requests/sec),18.04 - - Metric,avg,p50,p99 - Request Latency (ms),507.49,475.50,893.26 - Time to First Token (ms),77.82,71.99,137.26 - Inter Token Latency (ms),12.03,11.55,16.42 - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/fio.mdx b/doc/workloads/fio.mdx new file mode 100644 index 000000000..c3cb9814b --- /dev/null +++ b/doc/workloads/fio.mdx @@ -0,0 +1,120 @@ + + +`Fio` runs the open-source `fio` tool from CloudAI. It supports standalone and Slurm systems. + + + +## Scenarios + +Use `conf/common/test_scenario/fio.toml` for Slurm clusters. It runs the common `fio` test in a container and starts 4 tasks total: `num_nodes = 2` and `num_tasks_per_node = 2`. + +Use `conf/common/test_scenario/fio_local.toml` for local/standalone runs. It uses host-installed `fio` and no container image. + + + +## Fio Arguments + +Put fio CLI options under `cmd_args.args`. CloudAI does not declare a fixed fio option list; keys pass through verbatim. + +``` toml +name = "fio" +description = "fio random write smoke test" +test_template_name = "Fio" + +[cmd_args] +fio_binary = "fio" + + [cmd_args.args] + name = "fio-smoke" + filename = "/tmp/cloudai-fio-test" + rw = "randwrite" + bs = "128k" + size = "80m" + iodepth = 1 + numjobs = 1 + group_reporting = true + thread = true +``` + +This emits options like `--rw=randwrite` and `--group_reporting`. Quote TOML keys when option names contain characters such as `-`: + +``` toml +[cmd_args.args] +"max-jobs" = 4 +``` + +List values are CloudAI DSE sweeps: + +``` toml +[cmd_args.args] +bs = ["128k", "1m"] +iodepth = [1, 8, 32] +``` + +To repeat the same fio option in one run, use a nested table. Nested key names are only stable TOML item names; values become fio values: + +``` toml +[cmd_args.args.a] +"0" = "=foo" # --a==foo +"1" = "bar" # --a=bar + +[cmd_args.args."client"] +"0" = "host1" # --client=host1 +"1" = "host2" # --client=host2 +``` + +For existing or complex fio configs, use `job_file`: + +``` toml +[cmd_args] +fio_binary = "/tmp/fio/fio" +job_file = "/tmp/kv_emulation.fio" +``` + + + +## Slurm Tasks + +Scenario `num_nodes` controls node count. `cmd_args.num_tasks_per_node` controls Slurm tasks per node. Total fio tasks = `num_nodes * num_tasks_per_node`. CloudAI passes this as `--ntasks` and `--ntasks-per-node`. + +``` toml +[[Tests]] +id = "Tests.fio" +test_name = "fio" +num_nodes = 2 + + [Tests.cmd_args] + docker_image_url = "openeuler/fio:3.42-oe2403sp3" + num_tasks_per_node = 2 +``` + + + +## Default Metric + +For `agent_metrics = ["default"]`, CloudAI aggregates parsed fio summary rows. Defaults report total bandwidth across all operations and tasks. Bandwidth metrics are normalized to MiB/s before aggregation. Latency metrics are normalized to usec before aggregation. + +``` toml +[cmd_args] +metric_operation = "all" # read, write, trim, all, first +metric_name = "bw" # bw, iops, latency +metric_aggregate = "sum" # sum, mean, min, max, first +``` + +Raw parsed rows and normalized metric values are written to `fio_summary.csv`. + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/fio.rst b/doc/workloads/fio.rst deleted file mode 100644 index 820145a86..000000000 --- a/doc/workloads/fio.rst +++ /dev/null @@ -1,131 +0,0 @@ -fio -=== - -``Fio`` runs the open-source ``fio`` tool from CloudAI. -It supports standalone and Slurm systems. - -Scenarios ---------- - -Use ``conf/common/test_scenario/fio.toml`` for Slurm clusters. -It runs the common ``fio`` test in a container and starts 4 tasks total: -``num_nodes = 2`` and ``num_tasks_per_node = 2``. - -Use ``conf/common/test_scenario/fio_local.toml`` for local/standalone runs. -It uses host-installed ``fio`` and no container image. - -Fio Arguments -------------- - -Put fio CLI options under ``cmd_args.args``. -CloudAI does not declare a fixed fio option list; keys pass through verbatim. - -.. code-block:: toml - - name = "fio" - description = "fio random write smoke test" - test_template_name = "Fio" - - [cmd_args] - fio_binary = "fio" - - [cmd_args.args] - name = "fio-smoke" - filename = "/tmp/cloudai-fio-test" - rw = "randwrite" - bs = "128k" - size = "80m" - iodepth = 1 - numjobs = 1 - group_reporting = true - thread = true - -This emits options like ``--rw=randwrite`` and ``--group_reporting``. -Quote TOML keys when option names contain characters such as ``-``: - -.. code-block:: toml - - [cmd_args.args] - "max-jobs" = 4 - -List values are CloudAI DSE sweeps: - -.. code-block:: toml - - [cmd_args.args] - bs = ["128k", "1m"] - iodepth = [1, 8, 32] - -To repeat the same fio option in one run, use a nested table. -Nested key names are only stable TOML item names; values become fio values: - -.. code-block:: toml - - [cmd_args.args.a] - "0" = "=foo" # --a==foo - "1" = "bar" # --a=bar - - [cmd_args.args."client"] - "0" = "host1" # --client=host1 - "1" = "host2" # --client=host2 - -For existing or complex fio configs, use ``job_file``: - -.. code-block:: toml - - [cmd_args] - fio_binary = "/tmp/fio/fio" - job_file = "/tmp/kv_emulation.fio" - -Slurm Tasks ------------ - -Scenario ``num_nodes`` controls node count. -``cmd_args.num_tasks_per_node`` controls Slurm tasks per node. -Total fio tasks = ``num_nodes * num_tasks_per_node``. -CloudAI passes this as ``--ntasks`` and ``--ntasks-per-node``. - -.. code-block:: toml - - [[Tests]] - id = "Tests.fio" - test_name = "fio" - num_nodes = 2 - - [Tests.cmd_args] - docker_image_url = "openeuler/fio:3.42-oe2403sp3" - num_tasks_per_node = 2 - -Default Metric --------------- - -For ``agent_metrics = ["default"]``, CloudAI aggregates parsed fio summary rows. -Defaults report total bandwidth across all operations and tasks. -Bandwidth metrics are normalized to MiB/s before aggregation. -Latency metrics are normalized to usec before aggregation. - -.. code-block:: toml - - [cmd_args] - metric_operation = "all" # read, write, trim, all, first - metric_name = "bw" # bw, iops, latency - metric_aggregate = "sum" # sum, mean, min, max, first - -Raw parsed rows and normalized metric values are written to ``fio_summary.csv``. - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.fio.fio.FioCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.fio.fio.FioTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/index.mdx b/doc/workloads/index.mdx new file mode 100644 index 000000000..6597c4b59 --- /dev/null +++ b/doc/workloads/index.mdx @@ -0,0 +1,47 @@ + + +This chapter contains automatically generated documentation for all CloudAI workloads. Each workload provides specific functionality for running different types of tests and benchmarks. + + + +## Available Workloads + +| Test | Slurm | Kubernetes | RunAI | Standalone | +|----|----|----|----|----| +| [AIConfigurator](aiconfigurator.mdx) | ❌ | ❌ | ❌ | ✅ | +| [AI Dynamo](ai_dynamo.mdx) | ✅ | ✅ | ❌ | ❌ | +| [Bash Command](bash_cmd.mdx) | ✅ | ❌ | ❌ | ❌ | +| [Chakra Replay](chakra_replay.mdx) | ✅ | ❌ | ❌ | ❌ | +| [DDLB](ddlb.mdx) | ✅ | ❌ | ❌ | ❌ | +| [DeepEP Benchmark](deepep.mdx) | ✅ | ❌ | ❌ | ❌ | +| [DynamoMocker](dynamo_mocker.mdx) | ❌ | ❌ | ❌ | ✅ | +| [fio](fio.mdx) | ✅ | ❌ | ❌ | ✅ | +| [JaxToolbox workloads (DEPRECATED)](jax_toolbox.mdx) | ✅ | ❌ | ❌ | ❌ | +| MegatronRun | ✅ | ❌ | ❌ | ❌ | +| [MegatronBridge](megatron_bridge.mdx) | ✅ | ❌ | ❌ | ❌ | +| [NCCL](nccl.mdx) | ✅ | ✅ | ✅ | ❌ | +| [NeMo v1.0 aka NemoLauncher (DEPRECATED)](nemo_launcher.mdx) | ✅ | ❌ | ❌ | ❌ | +| [Nemo Run](nemo_run.mdx) | ✅ | ❌ | ❌ | ❌ | +| [NIXL Bench](nixl_bench.mdx) | ✅ | ❌ | ❌ | ❌ | +| [NIXL EP](nixl_ep.mdx) | ✅ | ❌ | ❌ | ❌ | +| [NIXL KVBench](nixl_kvbench.mdx) | ✅ | ❌ | ❌ | ❌ | +| [NIXL CTPerf](nixl_perftest.mdx) | ✅ | ❌ | ❌ | ❌ | +| [Sleep](sleep.mdx) | ✅ | ✅ | ❌ | ✅ | +| [SGLang](sglang.mdx) | ✅ | ❌ | ❌ | ❌ | +| [Slurm Container](slurm_container.mdx) | ✅ | ❌ | ❌ | ❌ | +| Triton Inference | ✅ | ❌ | ❌ | ❌ | +| [UCC](ucc.mdx) | ✅ | ❌ | ❌ | ❌ | +| [vLLM](vllm.mdx) | ✅ | ❌ | ❌ | ❌ | + + + +## Adding New Workloads + +To add documentation for a new workload: + +1. Add docstrings to your Python classes and methods. +2. Create an MDX file in `doc/workloads/` (e.g., `my_workload.mdx`). +3. Add it to the table above and to `fern/docs.yml`. +4. Add its reference classes to `fern/reference.yml` and include the generated snippets using Fern's `Markdown` component, with `src` set to `/generated/fully.qualified.Class.mdx`. + +The documentation will be automatically generated during the build process. diff --git a/doc/workloads/index.rst b/doc/workloads/index.rst deleted file mode 100644 index 916d596cc..000000000 --- a/doc/workloads/index.rst +++ /dev/null @@ -1,53 +0,0 @@ -Workloads -========= - -This chapter contains automatically generated documentation for all CloudAI workloads. Each workload provides specific functionality for running different types of tests and benchmarks. - -Available Workloads -------------------- - -.. csv-table:: - :header: "Test", "Slurm", "Kubernetes", "RunAI", "Standalone" - :widths: 40, 15, 15, 15, 15 - - ":doc:`aiconfigurator`", "❌", "❌", "❌", "✅" - ":doc:`ai_dynamo`", "✅", "✅", "❌", "❌" - ":doc:`bash_cmd`", "✅", "❌", "❌", "❌" - ":doc:`chakra_replay`", "✅", "❌", "❌", "❌" - ":doc:`ddlb`", "✅", "❌", "❌", "❌" - ":doc:`deepep`", "✅", "❌", "❌", "❌" - ":doc:`dynamo_mocker`", "❌", "❌", "❌", "✅" - ":doc:`fio`", "✅", "❌", "❌", "✅" - ":doc:`jax_toolbox`", "✅", "❌", "❌", "❌" - "MegatronRun", "✅", "❌", "❌", "❌" - ":doc:`megatron_bridge`", "✅", "❌", "❌", "❌" - ":doc:`nccl`", "✅", "✅", "✅", "❌" - ":doc:`nemo_launcher`", "✅", "❌", "❌", "❌" - ":doc:`nemo_run`", "✅", "❌", "❌", "❌" - ":doc:`nixl_bench`", "✅", "❌", "❌", "❌" - ":doc:`nixl_ep`", "✅", "❌", "❌", "❌" - ":doc:`nixl_kvbench`", "✅", "❌", "❌", "❌" - ":doc:`nixl_perftest`", "✅", "❌", "❌", "❌" - ":doc:`sleep`", "✅", "✅", "❌", "✅" - ":doc:`sglang`", "✅", "❌", "❌", "❌" - ":doc:`slurm_container`", "✅", "❌", "❌", "❌" - "Triton Inference", "✅", "❌", "❌", "❌" - ":doc:`ucc`", "✅", "❌", "❌", "❌" - ":doc:`vllm`", "✅", "❌", "❌", "❌" - -.. toctree:: - :hidden: - :glob: - - * - -Adding New Workloads ---------------------- - -To add documentation for a new workload: - -1. Add docstrings to your Python classes and methods. -2. Create a reStructuredText file in ``doc/workloads/`` (e.g., ``my_workload.rst``). -3. Add it to the table above. - -The documentation will be automatically generated during the build process. diff --git a/doc/workloads/jax_toolbox.mdx b/doc/workloads/jax_toolbox.mdx new file mode 100644 index 000000000..e47a2096d --- /dev/null +++ b/doc/workloads/jax_toolbox.mdx @@ -0,0 +1,28 @@ + + +This workload is **DEPRECATED**. + + + +## Troubleshooting Steps for Grok + +If an error occurs, follow these steps sequentially: + +1. **Read the Error Messages**: + Begin by reading the error messages printed by CloudAI. We strive to make our error messages clear and informative, so they are a good starting point for troubleshooting + +2. **Review profile_stderr.txt**: JaxToolbox operates in two stages: the profiling phase and the actual run phase. We follow the PGLE workflow as described in the [PGLE workflow documentation](https://github.com/google/paxml?tab=readme-ov-file#run-pgle-workflow-on-gpu). All stderr and stdout messages from the profiling phase are stored in `profile_stderr.txt`. If the profiling stage fails, you should find relevant error messages in this file. Attempt to understand the cause of the error from these messages. + +3. **Check the Actual Run Phase**: If the profiling stage completes successfully, CloudAI moves on to the actual run phase. The actual run generates stdout and stderr messages in separate files for each rank. Review these files to diagnose any issues during this phase. + + + +### Common Errors + +**DEADLINE_EXCEEDED**: +When running JaxToolbox on multiple nodes, the nodes must be able to communicate to execute a training job collaboratively. The DEADLINE_EXCEEDED error indicates a failure in the connection during the initialization stage. Potential causes include: + +- Hostname resolution failure by the slave nodes +- The port opened by the master node is not accessible by other nodes +- Network interface malfunctions +- Significant time gap in the initialization phase among nodes. If one node starts early while others are still loading the Docker image, this error can occur. This can happen when a Docker image is not locally cached, and all nodes try to download it from a remote registry without sufficient network bandwidth. The resulting difference in initialization times can lead to a timeout on some nodes diff --git a/doc/workloads/jax_toolbox.rst b/doc/workloads/jax_toolbox.rst deleted file mode 100644 index a064f3a53..000000000 --- a/doc/workloads/jax_toolbox.rst +++ /dev/null @@ -1,28 +0,0 @@ -JaxToolbox workloads (DEPRECATED) -================================= - -This workload is **DEPRECATED**. - -Troubleshooting Steps for Grok ------------------------------- - -If an error occurs, follow these steps sequentially: - -1. **Read the Error Messages**: - Begin by reading the error messages printed by CloudAI. We strive to make our error messages clear and informative, so they are a good starting point for troubleshooting - -2. **Review profile_stderr.txt**: JaxToolbox operates in two stages: the profiling phase and the actual run phase. We follow the PGLE workflow as described in the `PGLE workflow documentation `_. All stderr and stdout messages from the profiling phase are stored in ``profile_stderr.txt``. If the profiling stage fails, you should find relevant error messages in this file. Attempt to understand the cause of the error from these messages. - -3. **Check the Actual Run Phase**: - If the profiling stage completes successfully, CloudAI moves on to the actual run phase. The actual run generates stdout and stderr messages in separate files for each rank. Review these files to diagnose any issues during this phase. - -Common Errors -~~~~~~~~~~~~~ - -**DEADLINE_EXCEEDED**: - When running JaxToolbox on multiple nodes, the nodes must be able to communicate to execute a training job collaboratively. The DEADLINE_EXCEEDED error indicates a failure in the connection during the initialization stage. Potential causes include: - - - Hostname resolution failure by the slave nodes - - The port opened by the master node is not accessible by other nodes - - Network interface malfunctions - - Significant time gap in the initialization phase among nodes. If one node starts early while others are still loading the Docker image, this error can occur. This can happen when a Docker image is not locally cached, and all nodes try to download it from a remote registry without sufficient network bandwidth. The resulting difference in initialization times can lead to a timeout on some nodes diff --git a/doc/workloads/megatron_bridge.mdx b/doc/workloads/megatron_bridge.mdx new file mode 100644 index 000000000..b8939a59d --- /dev/null +++ b/doc/workloads/megatron_bridge.mdx @@ -0,0 +1,212 @@ + + +This workload (`test_template_name` is `MegatronBridge`) submits training and finetuning tasks based on Megatron-Bridge framework. + +Megatron-Bridge and NeMo Run generate the training script in dry-run mode on the submit host. CloudAI retains and parses their sbatch, then writes `cloudai_sbatch_script.sh` with its standard environment setup, metadata, rank mapping, and pre/post tests. These tests run inside the training allocation. Without the installed MBridge launcher or NeMo Run Python environment, `cloudai dry-run` produces a placeholder instead of the training commands and logs a warning. The placeholder exits with an error if executed. Install the dependencies and repeat the dry-run to generate the complete training script. + +CloudAI owns the job name, output/error paths, and node placement. Configured CloudAI allocation directives override matching NeMo Run directives; other NeMo Run directives and training artifact paths are retained. Set scenario `num_nodes` and system GPU/task counts to match the training configuration. `enable_vboost` uses CloudAI's standard VBoost setup in both normal and single-sbatch execution. An explicitly configured value takes precedence over the `ENABLE_VBOOST` environment setting. The rendezvous address uses NeMo's selected head-node hostname, which must resolve inside the training containers. + +Additional launcher options can be passed through `[extra_cmd_args]`. For example: + +``` toml +[extra_cmd_args] +"--custom_srun_args" = "--cpu-bind=none,--kill-on-bad-exit=1" +``` + +CloudAI combines `-cs` and `--custom_srun_args` from this table, then appends system and test-run `extra_srun_args` and pre-test-provided arguments. The result is passed through Megatron-Bridge's native `-cs` option without deduplication. + +For Nsight profiling, use Megatron-Bridge's `cmd_args.enable_nsys` and `profiling_start_step`, `profiling_stop_step`, and `profiling_ranks` options. The generic CloudAI `[nsys]` configuration is not applied to this workload. + + + +This workload has a hard requirement for the HuggingFace Hub token. There are two options: + +- (recommended) define `HF_TOKEN` environment variable +- set `cmd_args.hf_token` either in Test or Scenario config + + + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "megatron_bridge_qwen_30b" +description = "Megatron-Bridge run via CloudAI SlurmSystem for Qwen3 30B A3B" +test_template_name = "MegatronBridge" + +[[git_repos]] +url = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git" +commit = "v0.3.0" +mount_as = "/opt/Megatron-Bridge" + +[cmd_args] +gpu_type = "gb200" +gpus_per_node = 8 +num_gpus = 8 +# Container can be an NGC/enroot URL (nvcr.io#...) or a local .sqsh path. +container_image = "nvcr.io#nvidia/nemo:26.02.00" + +model_family_name = "qwen" +model_recipe_name = "qwen3_30b_a3b" +task = "pretrain" +domain = "llm" +compute_dtype = "fp8_mx" +``` + +Test Scenario example: + +``` toml +name = "megatron_bridge_qwen_30b" + +[[Tests]] +id = "megatron_bridge_qwen_30b" +test_name = "megatron_bridge_qwen_30b" +num_nodes = "2" +``` + +Test-in-Scenario example: + +``` toml +name = "megatron-bridge-test" + +[[Tests]] +id = "mbridge.1" +num_nodes = 2 +time_limit = "00:30:00" + +name = "megatron_bridge_qwen_30b" +description = "Megatron-Bridge run via CloudAI SlurmSystem for Qwen3 30B A3B" +test_template_name = "MegatronBridge" + + [[Tests.git_repos]] + url = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git" + commit = "v0.3.0" + mount_as = "/opt/Megatron-Bridge" + + [Tests.cmd_args] + container_image = "nvcr.io#nvidia/nemo:26.02.01" + model_family_name = "qwen" + model_recipe_name = "qwen3_30b_a3b" + + gpu_type = "gb200" + gpus_per_node = 8 + num_gpus = 8 + + task = "pretrain" + domain = "llm" + compute_dtype = "fp8_mx" +``` + + + +## Executor Environment Variables + +CloudAI applies `global_env_vars` and `extra_env_vars` through its standard job environment setup. These values override matching exports in the generated NeMo Run script. CloudAI's `env_vars.sh` is also sourced inside each training task, so configured values can reference task-local Slurm variables. + +Existing `cmd_args.executor_env_vars` entries are also accepted as job environment variables: + +``` toml +[cmd_args.executor_env_vars] +GPU_METRICS_NODES = "0,1" +``` + +`global_env_vars` and `extra_env_vars` take precedence over these entries. This does not require Megatron-Bridge's `-E/--env` support. + + + +## Chakra / Kineto Tracing + + + +### Disabling nsys + +The existing MegatronBridge TOMLs in `conf/staging/` have `enable_nsys = true`. To switch to Kineto/Chakra profiling, either: + +- **Omit** `enable_nsys` from the TOML (it defaults to `false`) +- **Or** explicitly set `enable_nsys = false` + +Both are equivalent. Do not set `enable_nsys = true` alongside Kineto flags — the two profilers conflict and Megatron-Bridge will raise an assertion error. + + + +### Enabling Kineto Trace + +Add the following to `[extra_cmd_args]` in your Test TOML. These are Hydra-style overrides passed directly to the Megatron-Bridge training script: + +``` toml +[extra_cmd_args] +"profiling.use_pytorch_profiler" = "true" +"profiling.profile_step_start" = "45" +"profiling.profile_step_end" = "50" +"profiling.profile_ranks" = "[0]" +``` + +This produces the following under the NemoRun experiment directory (`/experiments////`): + +- `torch_profile/rank-0.json.gz` — Chrome-format kineto trace (always produced) +- `tb_logs/` — TensorBoard events + + + +### Enabling Chakra Trace + +The Chakra trace is a PyTorch `ExecutionTraceObserver` recording — a separate file from the kineto trace. Add `pytorch_profiler_collect_chakra = true`: + +``` toml +[extra_cmd_args] +"profiling.use_pytorch_profiler" = "true" +"profiling.profile_step_start" = "45" +"profiling.profile_step_end" = "50" +"profiling.profile_ranks" = "[0]" +"profiling.pytorch_profiler_collect_chakra" = "true" +``` + +This produces an additional `chakra/rank-0.json.gz` in the same experiment directory. The kineto trace (`torch_profile/rank-0.json.gz`) is always produced regardless of this flag. + + + +### Reference TOML + +A ready-to-use TOML is provided at: + +``` text +conf/experimental/megatron_bridge/test/gb200/megatron_bridge_nemotron_nano_pytorch_profiler.toml +``` + +This runs Nemotron 3 Nano 3B (8 GPUs, 60 steps) with Kineto and Chakra profiling at steps 45–50 and is suitable for quickly validating the profiling setup. + + + +### Profiling Parameter Reference + +| Parameter | Type | Description | +|----|----|----| +| `profiling.use_pytorch_profiler` | bool | Enable PyTorch kineto profiler. Required for all Kineto/Chakra tracing. | +| `profiling.profile_step_start` | int | Training step at which profiling begins. | +| `profiling.profile_step_end` | int | Training step at which profiling ends and trace is exported. | +| `profiling.profile_ranks` | list\[int\] | GPU ranks to profile (e.g. `[0]` for rank 0 only). | +| `profiling.record_shapes` | bool | Record tensor shapes in the kineto trace. Expensive — off by default. | +| `profiling.record_memory_history` | bool | Record memory allocation history (produces `memory_profile.pickle`). Expensive — off by default. | +| `profiling.memory_snapshot_path` | str | Filename for the memory snapshot pickle file. | +| `profiling.pytorch_profiler_collect_chakra` | bool | Collect Chakra ExecutionTraceObserver trace (`chakra/rank-N.json.gz`). The kineto trace is always produced; this adds an additional file. | +| `profiling.pytorch_profiler_collect_callstack` | bool | Include Python call stack in the kineto trace (increases trace size). | +| `enable_nsys` | bool | Enable Nsight Systems profiling. **Cannot be used with Kineto.** Default: `false`. | + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/megatron_bridge.rst b/doc/workloads/megatron_bridge.rst deleted file mode 100644 index 830fe2728..000000000 --- a/doc/workloads/megatron_bridge.rst +++ /dev/null @@ -1,264 +0,0 @@ -MegatronBridge -============== - -This workload (`test_template_name` is ``MegatronBridge``) submits training and finetuning tasks based on Megatron-Bridge framework. - -Megatron-Bridge and NeMo Run generate the training script in dry-run mode on the -submit host. CloudAI retains and parses their sbatch, then writes -``cloudai_sbatch_script.sh`` with its standard environment setup, metadata, -rank mapping, and pre/post tests. These tests run inside the training allocation. -Without the installed MBridge launcher or NeMo Run Python environment, -``cloudai dry-run`` produces a placeholder instead of the training commands and -logs a warning. The placeholder exits with an error if executed. Install the -dependencies and repeat the dry-run to generate the complete training script. - -CloudAI owns the job name, output/error paths, and node placement. Configured -CloudAI allocation directives override matching NeMo Run directives; other -NeMo Run directives and training artifact paths are retained. Set scenario -``num_nodes`` and system GPU/task counts to match the training configuration. -``enable_vboost`` uses CloudAI's standard VBoost setup in both normal and single-sbatch execution. -An explicitly configured value takes precedence over the ``ENABLE_VBOOST`` environment setting. -The rendezvous address uses NeMo's selected head-node hostname, which must resolve inside the training containers. - -Additional launcher options can be passed through ``[extra_cmd_args]``. For example: - -.. code-block:: toml - - [extra_cmd_args] - "--custom_srun_args" = "--cpu-bind=none,--kill-on-bad-exit=1" - -CloudAI combines ``-cs`` and ``--custom_srun_args`` from this table, then appends -system and test-run ``extra_srun_args`` and pre-test-provided arguments. The result -is passed through Megatron-Bridge's native ``-cs`` option without deduplication. - -For Nsight profiling, use Megatron-Bridge's ``cmd_args.enable_nsys`` and -``profiling_start_step``, ``profiling_stop_step``, and ``profiling_ranks`` options. -The generic CloudAI ``[nsys]`` configuration is not applied to this workload. - -.. note:: - - This workload has a hard requirement for the HuggingFace Hub token. There are two options: - - - (recommended) define ``HF_TOKEN`` environment variable - - set ``cmd_args.hf_token`` either in Test or Scenario config - - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "megatron_bridge_qwen_30b" - description = "Megatron-Bridge run via CloudAI SlurmSystem for Qwen3 30B A3B" - test_template_name = "MegatronBridge" - - [[git_repos]] - url = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git" - commit = "v0.3.0" - mount_as = "/opt/Megatron-Bridge" - - [cmd_args] - gpu_type = "gb200" - gpus_per_node = 8 - num_gpus = 8 - # Container can be an NGC/enroot URL (nvcr.io#...) or a local .sqsh path. - container_image = "nvcr.io#nvidia/nemo:26.02.00" - - model_family_name = "qwen" - model_recipe_name = "qwen3_30b_a3b" - task = "pretrain" - domain = "llm" - compute_dtype = "fp8_mx" - -Test Scenario example: - -.. code-block:: toml - - name = "megatron_bridge_qwen_30b" - - [[Tests]] - id = "megatron_bridge_qwen_30b" - test_name = "megatron_bridge_qwen_30b" - num_nodes = "2" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "megatron-bridge-test" - - [[Tests]] - id = "mbridge.1" - num_nodes = 2 - time_limit = "00:30:00" - - name = "megatron_bridge_qwen_30b" - description = "Megatron-Bridge run via CloudAI SlurmSystem for Qwen3 30B A3B" - test_template_name = "MegatronBridge" - - [[Tests.git_repos]] - url = "https://github.com/NVIDIA-NeMo/Megatron-Bridge.git" - commit = "v0.3.0" - mount_as = "/opt/Megatron-Bridge" - - [Tests.cmd_args] - container_image = "nvcr.io#nvidia/nemo:26.02.01" - model_family_name = "qwen" - model_recipe_name = "qwen3_30b_a3b" - - gpu_type = "gb200" - gpus_per_node = 8 - num_gpus = 8 - - task = "pretrain" - domain = "llm" - compute_dtype = "fp8_mx" - -Executor Environment Variables ------------------------------- - -CloudAI applies ``global_env_vars`` and ``extra_env_vars`` through its standard -job environment setup. These values override matching exports in the generated -NeMo Run script. CloudAI's ``env_vars.sh`` is also sourced inside each training -task, so configured values can reference task-local Slurm variables. - -Existing ``cmd_args.executor_env_vars`` entries are also accepted as job -environment variables: - -.. code-block:: toml - - [cmd_args.executor_env_vars] - GPU_METRICS_NODES = "0,1" - -``global_env_vars`` and ``extra_env_vars`` take precedence over these entries. -This does not require Megatron-Bridge's ``-E/--env`` support. - -Chakra / Kineto Tracing ------------------------ - -Disabling nsys -~~~~~~~~~~~~~~ - -The existing MegatronBridge TOMLs in ``conf/staging/`` have ``enable_nsys = true``. -To switch to Kineto/Chakra profiling, either: - -- **Omit** ``enable_nsys`` from the TOML (it defaults to ``false``) -- **Or** explicitly set ``enable_nsys = false`` - -Both are equivalent. Do not set ``enable_nsys = true`` alongside Kineto flags — the -two profilers conflict and Megatron-Bridge will raise an assertion error. - - -Enabling Kineto Trace -~~~~~~~~~~~~~~~~~~~~~ - -Add the following to ``[extra_cmd_args]`` in your Test TOML. These are Hydra-style -overrides passed directly to the Megatron-Bridge training script: - -.. code-block:: toml - - [extra_cmd_args] - "profiling.use_pytorch_profiler" = "true" - "profiling.profile_step_start" = "45" - "profiling.profile_step_end" = "50" - "profiling.profile_ranks" = "[0]" - -This produces the following under the NemoRun experiment directory (``/experiments////``): - -- ``torch_profile/rank-0.json.gz`` — Chrome-format kineto trace (always produced) -- ``tb_logs/`` — TensorBoard events - - -Enabling Chakra Trace -~~~~~~~~~~~~~~~~~~~~~ - -The Chakra trace is a PyTorch ``ExecutionTraceObserver`` recording — a separate file -from the kineto trace. Add ``pytorch_profiler_collect_chakra = true``: - -.. code-block:: toml - - [extra_cmd_args] - "profiling.use_pytorch_profiler" = "true" - "profiling.profile_step_start" = "45" - "profiling.profile_step_end" = "50" - "profiling.profile_ranks" = "[0]" - "profiling.pytorch_profiler_collect_chakra" = "true" - -This produces an additional ``chakra/rank-0.json.gz`` in the same experiment directory. -The kineto trace (``torch_profile/rank-0.json.gz``) is always produced regardless of -this flag. - - -Reference TOML -~~~~~~~~~~~~~~ - -A ready-to-use TOML is provided at: - -.. code-block:: text - - conf/experimental/megatron_bridge/test/gb200/megatron_bridge_nemotron_nano_pytorch_profiler.toml - -This runs Nemotron 3 Nano 3B (8 GPUs, 60 steps) with Kineto and Chakra profiling at -steps 45–50 and is suitable for quickly validating the profiling setup. - - -Profiling Parameter Reference -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: 40 15 45 - - * - Parameter - - Type - - Description - * - ``profiling.use_pytorch_profiler`` - - bool - - Enable PyTorch kineto profiler. Required for all Kineto/Chakra tracing. - * - ``profiling.profile_step_start`` - - int - - Training step at which profiling begins. - * - ``profiling.profile_step_end`` - - int - - Training step at which profiling ends and trace is exported. - * - ``profiling.profile_ranks`` - - list[int] - - GPU ranks to profile (e.g. ``[0]`` for rank 0 only). - * - ``profiling.record_shapes`` - - bool - - Record tensor shapes in the kineto trace. Expensive — off by default. - * - ``profiling.record_memory_history`` - - bool - - Record memory allocation history (produces ``memory_profile.pickle``). Expensive — off by default. - * - ``profiling.memory_snapshot_path`` - - str - - Filename for the memory snapshot pickle file. - * - ``profiling.pytorch_profiler_collect_chakra`` - - bool - - Collect Chakra ExecutionTraceObserver trace (``chakra/rank-N.json.gz``). The kineto trace is always produced; this adds an additional file. - * - ``profiling.pytorch_profiler_collect_callstack`` - - bool - - Include Python call stack in the kineto trace (increases trace size). - * - ``enable_nsys`` - - bool - - Enable Nsight Systems profiling. **Cannot be used with Kineto.** Default: ``false``. - - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/nccl.mdx b/doc/workloads/nccl.mdx new file mode 100644 index 000000000..c5c1046b6 --- /dev/null +++ b/doc/workloads/nccl.mdx @@ -0,0 +1,67 @@ + + +This workload (`test_template_name` is `NcclTest`) allows users to execute NCCL benchmarks within the CloudAI framework. + + + +## Usage Example + +Test TOML example: + +``` toml +name = "my_nccl_test" +description = "Example NCCL test" +test_template_name = "NcclTest" + +[cmd_args] +docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" +``` + +Test Scenario example: + +``` toml +name = "nccl-test" + +[[Tests]] +id = "nccl.1" +num_nodes = 1 +time_limit = "00:05:00" + +test_name = "my_nccl_test" +``` + +Test-in-Scenario example: + +``` toml +name = "nccl-test" + +[[Tests]] +id = "nccl.1" +num_nodes = 1 +time_limit = "00:05:00" + +name = "my_nccl_test" +description = "Example NCCL test" +test_template_name = "NcclTest" + + [Tests.cmd_args] + docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" + subtest_name = "all_reduce_perf_mpi" + iters = 100 +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/nccl.rst b/doc/workloads/nccl.rst deleted file mode 100644 index 20502db95..000000000 --- a/doc/workloads/nccl.rst +++ /dev/null @@ -1,68 +0,0 @@ -NCCL -==== - -This workload (`test_template_name` is ``NcclTest``) allows users to execute NCCL benchmarks within the CloudAI framework. - -Usage Example -------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_nccl_test" - description = "Example NCCL test" - test_template_name = "NcclTest" - - [cmd_args] - docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" - -Test Scenario example: - -.. code-block:: toml - - name = "nccl-test" - - [[Tests]] - id = "nccl.1" - num_nodes = 1 - time_limit = "00:05:00" - - test_name = "my_nccl_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "nccl-test" - - [[Tests]] - id = "nccl.1" - num_nodes = 1 - time_limit = "00:05:00" - - name = "my_nccl_test" - description = "Example NCCL test" - test_template_name = "NcclTest" - - [Tests.cmd_args] - docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" - subtest_name = "all_reduce_perf_mpi" - iters = 100 - -API Documentation ---------------------------------- - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nccl_test.nccl.NCCLCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nccl_test.nccl.NCCLTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/nemo_launcher.mdx b/doc/workloads/nemo_launcher.mdx new file mode 100644 index 000000000..1d73097a3 --- /dev/null +++ b/doc/workloads/nemo_launcher.mdx @@ -0,0 +1,84 @@ + + +This workload is **DEPRECATED**. Please use [Nemo Run](nemo_run.mdx) instead. + + + +## Downloading and Installing the NeMo Dataset (The Pile Dataset) + +This section describes how users can download the NeMo datasets on the server. The install mode of CloudAI handles the installation of all test prerequisites, but downloading and installing datasets is not the responsibility of the install mode. This is because any large datasets should be installed globally by the administrator and shared with multiple users, even if a user does not use CloudAI. + +For CloudAI users, we provide a detailed guide about downloading and installing the NeMo datasets in this section. By default, the NeMo launcher uses mock datasets for testing purposes. If you want to run tests using real datasets, you must download the datasets and update the test `.toml` files accordingly to locate the datasets and provide appropriate prefixes. + +To understand the datasets available in the NeMo framework, you can refer to the Data Preparation section of [the document](https://docs.nvidia.com/launchpad/ai/base-command-nemo/latest/bc-nemo-step-02.html#use-bignlp-to-download-and-prepare-the-pile-dataset). According to the document, you can download and use the Pile dataset. The document also provides detailed instructions on how to download these datasets for various platforms. + +> Let’s assume that we have a Slurm cluster. + +You can download the datasets with the following command: + +``` bash +$ git clone https://github.com/NVIDIA/NeMo-Framework-Launcher.git +$ cd NeMo-Framework-Launcher +$ python3 launcher_scripts/main.py \ + container=nvcr.io/ea-bignlp/ga-participants/nemofw-training:23.11\ + stages=["data_preparation"]\ + launcher_scripts_path=$PWD/launcher_scripts\ + base_results_dir=$PWD/result\ + env_vars.TRANSFORMERS_OFFLINE=0\ + data_dir=directory_path_to_download_dataset\ + data_preparation.run.time_limit="96:00:00" +``` + +Once you submit a NeMo job with the data preparation stage, you should be able to find data downloading jobs with the `squeue` command. If this command does not work, please review the log files under `$PWD/result`. If you want to download the full Pile dataset, you should have at least 1TB of space in the directory to download the dataset because the Pile dataset size is 800GB. By default, NeMo will look at the configuration file under `conf/config.yaml`: + +``` yaml +defaults: + - data_preparation: baichuan2/download_baichuan2_pile + +stages: + - data_preparation +``` + +As the data preparation field points to `baichuan2/download_baichuan2_pile`, it will read the YAML file: + +``` yaml +run: + name: download_baichuan2_pile + results_dir: ${base_results_dir}/${.name} + time_limit: "4:00:00" + dependency: "singleton" + node_array_size: 30 + array: ${..file_numbers} + bcp_preproc_npernode: 2 # 2 should be safe to use and x2 times faster. + +dataset: pile +download_the_pile: True # Whether to download the pile dataset from the internet. +the_pile_url: "https://huggingface.co/datasets/monology/pile-uncopyrighted/resolve/main/train/" # Source URL to download The Pile dataset from. +file_numbers: "0-29" # The pile dataset consists of 30 files (0-29), choose which ones to download. +preprocess_data: True # True to preprocess the data from a jsonl file, False otherwise. +download_tokenizer_url: "https://huggingface.co/baichuan-inc/Baichuan2-7B-Base/blob/main/tokenizer.model" +tokenizer_typzer_library: "sentencepiece" +tokenizer_save_dir: ${data_dir}/baichuan2 +tokenizer_model: ${.tokenizer_save_dir}/baichuan2_tokenizer.model +rm_downloaded: False # Extract script will remove downloaded zst after extraction +rm_extracted: False # Preprocess script will remove extracted files after preproc. +``` + +You can update the fields to adjust the behavior. For example, you can update the `file_numbers` field to adjust the number of dataset files to download. This will allow you to save disk space. + + + +## Note: For running Nemo Llama model + +It is important to follow these additional steps: + +1. Go to [🤗 Hugging Face](https://huggingface.co/docs/transformers/en/model_doc/llama). +2. Follow the instructions on how to download the tokenizer. +3. Replace `TOKENIZER_MODEL` in `training.model.tokenizer.model=TOKENIZER_MODEL` with your path (the tokenizer should be a `.model` file) in `conf/common/test/llama.toml`. + + + +## Troubleshooting + +- If your run is not successful, please review the stderr and stdout files generated under the results directory. Within the output directory, locate the run directory, and under the run directory, you will find stderr files like `log-nemo-megatron-run_[job_id].err`. Please review these files for any meaningful error messages +- Trying the CloudAI-generated NeMo launcher command can be helpful as well. You can find the executed command in your stdout and in your log file (debug.log) in your current working directory. Review and run the command, and you can modify the arguments to troubleshoot the issue diff --git a/doc/workloads/nemo_launcher.rst b/doc/workloads/nemo_launcher.rst deleted file mode 100644 index daf0576ed..000000000 --- a/doc/workloads/nemo_launcher.rst +++ /dev/null @@ -1,83 +0,0 @@ -NeMo v1.0 aka NemoLauncher (DEPRECATED) -======================================= -This workload is **DEPRECATED**. Please use :doc:`nemo_run` instead. - - -Downloading and Installing the NeMo Dataset (The Pile Dataset) --------------------------------------------------------------- -This section describes how users can download the NeMo datasets on the server. The install mode of CloudAI handles the installation of all test prerequisites, but downloading and installing datasets is not the responsibility of the install mode. This is because any large datasets should be installed globally by the administrator and shared with multiple users, even if a user does not use CloudAI. - -For CloudAI users, we provide a detailed guide about downloading and installing the NeMo datasets in this section. By default, the NeMo launcher uses mock datasets for testing purposes. If you want to run tests using real datasets, you must download the datasets and update the test `.toml` files accordingly to locate the datasets and provide appropriate prefixes. - -To understand the datasets available in the NeMo framework, you can refer to the Data Preparation section of `the document `_. According to the document, you can download and use the Pile dataset. The document also provides detailed instructions on how to download these datasets for various platforms. - - Let’s assume that we have a Slurm cluster. - -You can download the datasets with the following command: - -.. code-block:: bash - - $ git clone https://github.com/NVIDIA/NeMo-Framework-Launcher.git - $ cd NeMo-Framework-Launcher - $ python3 launcher_scripts/main.py \ - container=nvcr.io/ea-bignlp/ga-participants/nemofw-training:23.11\ - stages=["data_preparation"]\ - launcher_scripts_path=$PWD/launcher_scripts\ - base_results_dir=$PWD/result\ - env_vars.TRANSFORMERS_OFFLINE=0\ - data_dir=directory_path_to_download_dataset\ - data_preparation.run.time_limit="96:00:00" - -Once you submit a NeMo job with the data preparation stage, you should be able to find data downloading jobs with the ``squeue`` command. If this command does not work, please review the log files under ``$PWD/result``. If you want to download the full Pile dataset, you should have at least 1TB of space in the directory to download the dataset because the Pile dataset size is 800GB. -By default, NeMo will look at the configuration file under ``conf/config.yaml``: - -.. code-block:: yaml - - defaults: - - data_preparation: baichuan2/download_baichuan2_pile - - stages: - - data_preparation - -As the data preparation field points to ``baichuan2/download_baichuan2_pile``, it will read the YAML file: - -.. code-block:: yaml - - run: - name: download_baichuan2_pile - results_dir: ${base_results_dir}/${.name} - time_limit: "4:00:00" - dependency: "singleton" - node_array_size: 30 - array: ${..file_numbers} - bcp_preproc_npernode: 2 # 2 should be safe to use and x2 times faster. - - dataset: pile - download_the_pile: True # Whether to download the pile dataset from the internet. - the_pile_url: "https://huggingface.co/datasets/monology/pile-uncopyrighted/resolve/main/train/" # Source URL to download The Pile dataset from. - file_numbers: "0-29" # The pile dataset consists of 30 files (0-29), choose which ones to download. - preprocess_data: True # True to preprocess the data from a jsonl file, False otherwise. - download_tokenizer_url: "https://huggingface.co/baichuan-inc/Baichuan2-7B-Base/blob/main/tokenizer.model" - tokenizer_typzer_library: "sentencepiece" - tokenizer_save_dir: ${data_dir}/baichuan2 - tokenizer_model: ${.tokenizer_save_dir}/baichuan2_tokenizer.model - rm_downloaded: False # Extract script will remove downloaded zst after extraction - rm_extracted: False # Preprocess script will remove extracted files after preproc. - -You can update the fields to adjust the behavior. For example, you can update the ``file_numbers`` field to adjust the number of dataset files to download. This will allow you to save disk space. - -Note: For running Nemo Llama model ------------------------------------- - -It is important to follow these additional steps: - -1. Go to `🤗 Hugging Face `_. -2. Follow the instructions on how to download the tokenizer. -3. Replace ``TOKENIZER_MODEL`` in ``training.model.tokenizer.model=TOKENIZER_MODEL`` with your path (the tokenizer should be a ``.model`` file) in ``conf/common/test/llama.toml``. - - -Troubleshooting ---------------- - -* If your run is not successful, please review the stderr and stdout files generated under the results directory. Within the output directory, locate the run directory, and under the run directory, you will find stderr files like ``log-nemo-megatron-run_[job_id].err``. Please review these files for any meaningful error messages -* Trying the CloudAI-generated NeMo launcher command can be helpful as well. You can find the executed command in your stdout and in your log file (debug.log) in your current working directory. Review and run the command, and you can modify the arguments to troubleshoot the issue diff --git a/doc/workloads/nemo_run.mdx b/doc/workloads/nemo_run.mdx new file mode 100644 index 000000000..489bdca5e --- /dev/null +++ b/doc/workloads/nemo_run.mdx @@ -0,0 +1,67 @@ + + +This workload (`test_template_name` is `NemoRun`) executes NeMo training and fine-tuning tasks using the NeMo Run framework. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_nemo_test" +description = "Example NeMo Run test" +test_template_name = "NemoRun" + +[cmd_args] +recipe = "llama3_8b" +task = "pretrain" +``` + +Test Scenario example: + +``` toml +name = "nemo-run-test" + +[[Tests]] +id = "nemo.1" +num_nodes = 4 +time_limit = "02:00:00" + +test_name = "my_nemo_test" +``` + +Test-in-Scenario example: + +``` toml +name = "nemo-run-test" + +[[Tests]] +id = "nemo.1" +num_nodes = 4 +time_limit = "02:00:00" + +name = "my_nemo_test" +description = "Example NeMo Run test" +test_template_name = "NemoRun" + + [Tests.cmd_args] + recipe = "llama3_8b" + task = "pretrain" +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/nemo_run.rst b/doc/workloads/nemo_run.rst deleted file mode 100644 index 0eb0eace7..000000000 --- a/doc/workloads/nemo_run.rst +++ /dev/null @@ -1,68 +0,0 @@ -Nemo Run -======== - -This workload (`test_template_name` is ``NemoRun``) executes NeMo training and fine-tuning tasks using the NeMo Run framework. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_nemo_test" - description = "Example NeMo Run test" - test_template_name = "NemoRun" - - [cmd_args] - recipe = "llama3_8b" - task = "pretrain" - -Test Scenario example: - -.. code-block:: toml - - name = "nemo-run-test" - - [[Tests]] - id = "nemo.1" - num_nodes = 4 - time_limit = "02:00:00" - - test_name = "my_nemo_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "nemo-run-test" - - [[Tests]] - id = "nemo.1" - num_nodes = 4 - time_limit = "02:00:00" - - name = "my_nemo_test" - description = "Example NeMo Run test" - test_template_name = "NemoRun" - - [Tests.cmd_args] - recipe = "llama3_8b" - task = "pretrain" - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nemo_run.nemo_run.NeMoRunCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nemo_run.nemo_run.NeMoRunTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/nixl_bench.mdx b/doc/workloads/nixl_bench.mdx new file mode 100644 index 000000000..d09d42cd0 --- /dev/null +++ b/doc/workloads/nixl_bench.mdx @@ -0,0 +1,138 @@ + + +This workload (`test_template_name` is `NIXLBench`) runs NIXL benchmarking suite for network and interconnect performance testing. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_nixl_bench_test" +description = "Example NIXL Bench test" +test_template_name = "NIXLBench" + +[cmd_args] +docker_image_url = "" +path_to_benchmark = "/workspace/nixlbench/build/nixlbench" +backend = "UCX" +initiator_seg_type = "VRAM" +target_seg_type = "VRAM" +op_type = "READ" +filepath = "/data" +device_list = "11:F:/store0.bin" +# one could also use kb, mb, gb shortcuts +total_buffer_size = 8000000000 +``` + +Test Scenario example: + +``` toml +name = "nixl-bench-test" + +[[Tests]] +id = "bench.1" +num_nodes = 1 +time_limit = "00:10:00" + +test_name = "my_nixl_bench_test" +``` + +Test-in-Scenario example: + +``` toml +name = "nixl-bench-test" + +[[Tests]] +id = "bench.1" +num_nodes = 1 +time_limit = "00:10:00" + +name = "my_nixl_bench_test" +description = "Example NIXL Bench test" +test_template_name = "NIXLBench" + + [Tests.cmd_args] + docker_image_url = "" + path_to_benchmark = "/workspace/nixlbench/build/nixlbench" + backend = "UCX" + initiator_seg_type = "DRAM" + target_seg_type = "DRAM" + op_type = "WRITE" +``` + + + +## Runtime Coordination + +NIXLBench uses ETCD by default. CloudAI starts ETCD from the benchmark image when `etcd_image_url` is omitted, or from the configured image. + +To use NIXLBench's direct two-process ASIO runtime instead, set: + +``` toml +runtime_type = "ASIO" +``` + +CloudAI resolves `asio_address` to the first allocated node by default and does not install or start ETCD. `asio_address` and `asio_port` can be overridden explicitly. ASIO requires exactly two NIXLBench processes. For UCX, a one-node test runs both processes locally, while a two-node test runs one process on each node. + +Storage backends can run without either runtime by using an empty ETCD endpoint: + +``` toml +backend = "POSIX" +etcd_endpoints = "" +``` + +This null-runtime mode is limited to storage backends and launches one NIXLBench process by default. + + + +## Independent Storage Processes + +To run independent storage benchmarks in one allocation, set `launch_mode = "independent"` and `etcd_endpoints = ""`. CloudAI launches one process per node by default; use `num_nodes` to choose the number of nodes: + +``` toml +name = "storage-scaling" + +[[Tests]] +id = "storage.4nodes" +test_name = "my_storage_test" +num_nodes = 4 + + [Tests.cmd_args] + launch_mode = "independent" + etcd_endpoints = "" + num_threads = [4, 8] +``` + +Here, `my_storage_test` is a NIXLBench test defining the storage backend, container image, benchmark path and workload arguments. CloudAI runs the 4-thread and 8-thread configurations as separate tests. + +`launch_mode` selects CloudAI's launch strategy and is not passed to NIXLBench. Its default value, `"default"`, preserves existing behavior, including when task placement options are present. Placement options alone do not enable independent runs. + +Independent mode requires `etcd_endpoints = ""` and the default `runtime_type = "ETCD"` setting, which together select NIXLBench's null runtime. Managed/external ETCD and ASIO cannot be used with this mode. CloudAI launches one Slurm step across the requested nodes. Use `extra_srun_args` only when overriding the default task placement. System-level options precede test-level options. + +Each task executes in its own shell. Node-specific environment values and command expansions are evaluated there. CloudAI does not discover backend devices. + + + +### Results and Reporting + +Independent runs retain `nixlbench/.stdout`, `.stderr`, `.status` and `.hostname` files. The `nixlbench/ntasks` file records the actual Slurm task count. A failed process, missing task output or inconsistent block-size/batch-size coverage makes the run unsuccessful; incomplete results are not aggregated. + +`nixlbench_per_task.csv` preserves individual measurements with task IDs and hostnames. `nixlbench.csv` summarizes matching block sizes and batch sizes: `avg_lat` and `bw_gb_sec` are arithmetic means across tasks, `bw_min_gb_sec` is the minimum task bandwidth, `bw_sum_gb_sec` is the sum, and `task_count` records the contributors. The native HTML report shows mean latency and minimum/mean/summed bandwidth, with separate plots for each batch size. Comparison reports and metric observations use the mean per-task bandwidth and latency. Existing single-process reports are unchanged. + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/nixl_bench.rst b/doc/workloads/nixl_bench.rst deleted file mode 100644 index f75997eaa..000000000 --- a/doc/workloads/nixl_bench.rst +++ /dev/null @@ -1,159 +0,0 @@ -NIXL Bench -========== - -This workload (`test_template_name` is ``NIXLBench``) runs NIXL benchmarking suite for network and interconnect performance testing. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_nixl_bench_test" - description = "Example NIXL Bench test" - test_template_name = "NIXLBench" - - [cmd_args] - docker_image_url = "" - path_to_benchmark = "/workspace/nixlbench/build/nixlbench" - backend = "UCX" - initiator_seg_type = "VRAM" - target_seg_type = "VRAM" - op_type = "READ" - filepath = "/data" - device_list = "11:F:/store0.bin" - # one could also use kb, mb, gb shortcuts - total_buffer_size = 8000000000 - -Test Scenario example: - -.. code-block:: toml - - name = "nixl-bench-test" - - [[Tests]] - id = "bench.1" - num_nodes = 1 - time_limit = "00:10:00" - - test_name = "my_nixl_bench_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "nixl-bench-test" - - [[Tests]] - id = "bench.1" - num_nodes = 1 - time_limit = "00:10:00" - - name = "my_nixl_bench_test" - description = "Example NIXL Bench test" - test_template_name = "NIXLBench" - - [Tests.cmd_args] - docker_image_url = "" - path_to_benchmark = "/workspace/nixlbench/build/nixlbench" - backend = "UCX" - initiator_seg_type = "DRAM" - target_seg_type = "DRAM" - op_type = "WRITE" - -Runtime Coordination --------------------- - -NIXLBench uses ETCD by default. CloudAI starts ETCD from the benchmark image when -``etcd_image_url`` is omitted, or from the configured image. - -To use NIXLBench's direct two-process ASIO runtime instead, set: - -.. code-block:: toml - - runtime_type = "ASIO" - -CloudAI resolves ``asio_address`` to the first allocated node by default and does not -install or start ETCD. ``asio_address`` and ``asio_port`` can be overridden explicitly. -ASIO requires exactly two NIXLBench processes. For UCX, a one-node test runs both -processes locally, while a two-node test runs one process on each node. - -Storage backends can run without either runtime by using an empty ETCD endpoint: - -.. code-block:: toml - - backend = "POSIX" - etcd_endpoints = "" - -This null-runtime mode is limited to storage backends and launches one NIXLBench process by default. - -Independent Storage Processes ------------------------------ - -To run independent storage benchmarks in one allocation, set -``launch_mode = "independent"`` and ``etcd_endpoints = ""``. CloudAI launches one -process per node by default; use ``num_nodes`` to choose the number of nodes: - -.. code-block:: toml - - name = "storage-scaling" - - [[Tests]] - id = "storage.4nodes" - test_name = "my_storage_test" - num_nodes = 4 - - [Tests.cmd_args] - launch_mode = "independent" - etcd_endpoints = "" - num_threads = [4, 8] - -Here, ``my_storage_test`` is a NIXLBench test defining the storage backend, container -image, benchmark path and workload arguments. CloudAI runs the 4-thread and 8-thread -configurations as separate tests. - -``launch_mode`` selects CloudAI's launch strategy and is not passed to NIXLBench. -Its default value, ``"default"``, preserves existing behavior, including when task -placement options are present. Placement options alone do not enable independent runs. - -Independent mode requires ``etcd_endpoints = ""`` and the default -``runtime_type = "ETCD"`` setting, which together select NIXLBench's null runtime. -Managed/external ETCD and ASIO cannot be used with this mode. CloudAI launches one -Slurm step across the requested nodes. Use ``extra_srun_args`` only when overriding -the default task placement. System-level options precede test-level options. - -Each task executes in its own shell. Node-specific environment values and command -expansions are evaluated there. CloudAI does not discover backend devices. - -Results and Reporting -~~~~~~~~~~~~~~~~~~~~~ - -Independent runs retain ``nixlbench/.stdout``, ``.stderr``, ``.status`` and -``.hostname`` files. The ``nixlbench/ntasks`` file records the actual Slurm task count. -A failed process, missing task output or inconsistent block-size/batch-size coverage -makes the run unsuccessful; incomplete results are not aggregated. - -``nixlbench_per_task.csv`` preserves individual measurements with task IDs and hostnames. -``nixlbench.csv`` summarizes matching block sizes and batch sizes: ``avg_lat`` and -``bw_gb_sec`` are arithmetic means across tasks, ``bw_min_gb_sec`` is the minimum -task bandwidth, ``bw_sum_gb_sec`` is the sum, and ``task_count`` records the contributors. -The native HTML report shows mean latency and minimum/mean/summed bandwidth, with -separate plots for each batch size. Comparison reports and metric observations use -the mean per-task bandwidth and latency. Existing single-process reports are unchanged. - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchCmdArgs - :members: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/nixl_ep.mdx b/doc/workloads/nixl_ep.mdx new file mode 100644 index 000000000..39eae3c16 --- /dev/null +++ b/doc/workloads/nixl_ep.mdx @@ -0,0 +1,109 @@ + + +This workload (`test_template_name` is `NixlEP`) runs the NIXL Elastic EP benchmark through a Slurm-managed multi-node elastic launcher flow. + + + +## Overview + +The Slurm launch model is: + +- one `elastic.py` process per node, started in sequence as the plan progresses +- the master node starts first and exposes a TCPStore for rank coordination +- follower nodes connect via `--tcp-server $master_ip` once the master is ready +- the benchmark runtime comes from the container image +- each run serializes its plan JSON into the output directory + + + +## Plan Format + +The `plan` field is a JSON-encoded list of phases. Each phase is a list of rank indices passed directly to the benchmark. CloudAI uses the following convention to drive the elastic launcher: + +- **Positive rank index** — the rank is active. A rank that is new relative to the previous phase causes CloudAI to fire an additional `srun` for that worker. +- **Negative rank index** (e.g. `-6`) — signals a contraction: the benchmark sees the absolute value and treats it as temporarily removed. No new `srun` is launched for negative indices. +- **Omitted rank** — a rank present in an earlier phase but absent from the current phase list is not relaunched. The benchmark's own phase logic handles its inactivity. + +Example: + +``` text +[[0, 1, 2, 3], # phase 0: ranks 0–3 start + [0, 1, 2, 3, 4, 5, 6, 7], # phase 1: ranks 4–7 join (expansion) + [0, 1, 2, 3, 4, -6, 7], # phase 2: rank 6 contracted (no new launch) + [0, 1, 2, 3, 4, 5, 6, 7]] # phase 3: rank 6 rejoins (new launch for rank 6) +``` + +Phase completion is detected by polling the primary log for `-> end phase N` markers. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "nixl-ep-expansion-contraction" +description = "NIXL Elastic EP expansion/contraction benchmark" +test_template_name = "NixlEP" + +[cmd_args] +docker_image_url = "" +elastic_script = "/workspace/nixl/examples/device/ep/tests/elastic/elastic.py" +plan = "[[0, 1, 2, 3], [0, 1, 2, 3, 4, 5, 6, 7], [0, 1, 2, 3, 4, -6, 7], [0, 1, 2, 3, 4, 5, 6, 7]]" +num_processes_per_node = 4 +num_tokens = 256 +num_experts_per_rank = 4 +hidden_dim = 8192 +num_topk = 6 +disable_ll_nvlink = true +``` + +Test-in-Scenario example: + +``` toml +name = "nixl-ep-expansion-contraction" + +[[Tests]] +id = "nixl_ep.expansion_contraction" +num_nodes = 3 +time_limit = "00:30:00" + +name = "nixl-ep-expansion-contraction" +description = "NIXL Elastic EP expansion/contraction benchmark" +test_template_name = "NixlEP" + + [Tests.cmd_args] + docker_image_url = "" + elastic_script = "/workspace/nixl/examples/device/ep/tests/elastic/elastic.py" + plan = "[[0, 1, 2, 3], [0, 1, 2, 3, 4, 5, 6, 7], [0, 1, 2, 3, 4, -6, 7], [0, 1, 2, 3, 4, 5, 6, 7]]" + num_processes_per_node = 4 + num_tokens = 256 + num_experts_per_rank = 4 + hidden_dim = 8192 + num_topk = 6 + disable_ll_nvlink = true +``` + + + +## Reporting + +After a run completes, CloudAI prints a single table with one row per (node, rank) measurement. The `Phases` column shows each phase index colour-coded green (passed) or red (failed). Bandwidth columns report dispatch+combine throughput and timing per rank. + +The reported metric (`default`) is the mean dispatch+combine bandwidth in GB/s across all ranks. + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/nixl_ep.rst b/doc/workloads/nixl_ep.rst deleted file mode 100644 index 162c3a06b..000000000 --- a/doc/workloads/nixl_ep.rst +++ /dev/null @@ -1,106 +0,0 @@ -NIXL EP -======= - -This workload (``test_template_name`` is ``NixlEP``) runs the NIXL Elastic EP benchmark through a Slurm-managed multi-node elastic launcher flow. - -Overview --------- - -The Slurm launch model is: - -- one ``elastic.py`` process per node, started in sequence as the plan progresses -- the master node starts first and exposes a TCPStore for rank coordination -- follower nodes connect via ``--tcp-server $master_ip`` once the master is ready -- the benchmark runtime comes from the container image -- each run serializes its plan JSON into the output directory - -Plan Format ------------ - -The ``plan`` field is a JSON-encoded list of phases. Each phase is a list of rank indices passed directly to the benchmark. CloudAI uses the following convention to drive the elastic launcher: - -- **Positive rank index** — the rank is active. A rank that is new relative to the previous phase causes CloudAI to fire an additional ``srun`` for that worker. -- **Negative rank index** (e.g. ``-6``) — signals a contraction: the benchmark sees the absolute value and treats it as temporarily removed. No new ``srun`` is launched for negative indices. -- **Omitted rank** — a rank present in an earlier phase but absent from the current phase list is not relaunched. The benchmark's own phase logic handles its inactivity. - -Example: - -.. code-block:: text - - [[0, 1, 2, 3], # phase 0: ranks 0–3 start - [0, 1, 2, 3, 4, 5, 6, 7], # phase 1: ranks 4–7 join (expansion) - [0, 1, 2, 3, 4, -6, 7], # phase 2: rank 6 contracted (no new launch) - [0, 1, 2, 3, 4, 5, 6, 7]] # phase 3: rank 6 rejoins (new launch for rank 6) - -Phase completion is detected by polling the primary log for ``-> end phase N`` markers. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "nixl-ep-expansion-contraction" - description = "NIXL Elastic EP expansion/contraction benchmark" - test_template_name = "NixlEP" - - [cmd_args] - docker_image_url = "" - elastic_script = "/workspace/nixl/examples/device/ep/tests/elastic/elastic.py" - plan = "[[0, 1, 2, 3], [0, 1, 2, 3, 4, 5, 6, 7], [0, 1, 2, 3, 4, -6, 7], [0, 1, 2, 3, 4, 5, 6, 7]]" - num_processes_per_node = 4 - num_tokens = 256 - num_experts_per_rank = 4 - hidden_dim = 8192 - num_topk = 6 - disable_ll_nvlink = true - -Test-in-Scenario example: - -.. code-block:: toml - - name = "nixl-ep-expansion-contraction" - - [[Tests]] - id = "nixl_ep.expansion_contraction" - num_nodes = 3 - time_limit = "00:30:00" - - name = "nixl-ep-expansion-contraction" - description = "NIXL Elastic EP expansion/contraction benchmark" - test_template_name = "NixlEP" - - [Tests.cmd_args] - docker_image_url = "" - elastic_script = "/workspace/nixl/examples/device/ep/tests/elastic/elastic.py" - plan = "[[0, 1, 2, 3], [0, 1, 2, 3, 4, 5, 6, 7], [0, 1, 2, 3, 4, -6, 7], [0, 1, 2, 3, 4, 5, 6, 7]]" - num_processes_per_node = 4 - num_tokens = 256 - num_experts_per_rank = 4 - hidden_dim = 8192 - num_topk = 6 - disable_ll_nvlink = true - -Reporting ---------- - -After a run completes, CloudAI prints a single table with one row per (node, rank) measurement. The ``Phases`` column shows each phase index colour-coded green (passed) or red (failed). Bandwidth columns report dispatch+combine throughput and timing per rank. - -The reported metric (``default``) is the mean dispatch+combine bandwidth in GB/s across all ranks. - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.nixl_ep.nixl_ep.NixlEPCmdArgs - :members: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nixl_ep.nixl_ep.NixlEPTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/nixl_kvbench.mdx b/doc/workloads/nixl_kvbench.mdx new file mode 100644 index 000000000..f50283898 --- /dev/null +++ b/doc/workloads/nixl_kvbench.mdx @@ -0,0 +1,76 @@ + + +This workload (`test_template_name` is `NIXLKVBench`) runs NIXL KV-cache benchmarking for key-value store performance testing. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_nixl_kvbench_test" +description = "Example NIXL KVBench test" +test_template_name = "NIXLKVBench" + +[cmd_args] +docker_image_url = "" +model = "./examples/model_deepseek_r1.yaml" +model_config = "./examples/block-tp1-pp16.yaml" +backend = "POSIX" +num_requests = 1 +source = "file" +num_iter = 16 +page_size = 256 +filepath = "/data" +``` + +Test Scenario example: + +``` toml +name = "nixl-kvbench-test" + +[[Tests]] +id = "kvbench.1" +num_nodes = 1 +time_limit = "00:10:00" + +test_name = "my_nixl_kvbench_test" +``` + +Test-in-Scenario example: + +``` toml +name = "nixl-kvbench-test" + +[[Tests]] +id = "kvbench.1" +num_nodes = 1 +time_limit = "00:10:00" + +name = "my_nixl_kvbench_test" +description = "Example NIXL KVBench test" +test_template_name = "NIXLKVBench" + + [Tests.cmd_args] + docker_image_url = "" + backend = "UCX" + source = "memory" + op_type = "READ" +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/nixl_kvbench.rst b/doc/workloads/nixl_kvbench.rst deleted file mode 100644 index 310211e8a..000000000 --- a/doc/workloads/nixl_kvbench.rst +++ /dev/null @@ -1,76 +0,0 @@ -NIXL KVBench -============ - -This workload (`test_template_name` is ``NIXLKVBench``) runs NIXL KV-cache benchmarking for key-value store performance testing. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_nixl_kvbench_test" - description = "Example NIXL KVBench test" - test_template_name = "NIXLKVBench" - - [cmd_args] - docker_image_url = "" - model = "./examples/model_deepseek_r1.yaml" - model_config = "./examples/block-tp1-pp16.yaml" - backend = "POSIX" - num_requests = 1 - source = "file" - num_iter = 16 - page_size = 256 - filepath = "/data" - -Test Scenario example: - -.. code-block:: toml - - name = "nixl-kvbench-test" - - [[Tests]] - id = "kvbench.1" - num_nodes = 1 - time_limit = "00:10:00" - - test_name = "my_nixl_kvbench_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "nixl-kvbench-test" - - [[Tests]] - id = "kvbench.1" - num_nodes = 1 - time_limit = "00:10:00" - - name = "my_nixl_kvbench_test" - description = "Example NIXL KVBench test" - test_template_name = "NIXLKVBench" - - [Tests.cmd_args] - docker_image_url = "" - backend = "UCX" - source = "memory" - op_type = "READ" - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchCmdArgs - :members: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/nixl_perftest.mdx b/doc/workloads/nixl_perftest.mdx new file mode 100644 index 000000000..51a12e292 --- /dev/null +++ b/doc/workloads/nixl_perftest.mdx @@ -0,0 +1,86 @@ + + +This workload (`test_template_name` is `NixlPerftest`) runs NIXL performance testing suite for comprehensive network performance evaluation. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_nixl_perftest_test" +description = "Example NIXL Perftest test" +test_template_name = "NixlPerftest" + +[cmd_args] +docker_image_url = "" +subtest = "sequential-ct-perftest" +num_user_requests = 1 +batch_size = 1 +num_prefill_nodes = 1 +num_decode_nodes = 1 +prefill_tp = 4 +decode_tp = 4 +isl_mean = 10000 +isl_scale = 3000 +model = "deepseek-r1-distill-llama-70b" +``` + +Test Scenario example: + +``` toml +name = "nixl-perftest-test" + +[[Tests]] +id = "perftest.1" +num_nodes = 2 +time_limit = "00:20:00" + +test_name = "my_nixl_perftest_test" +``` + +Test-in-Scenario example: + +``` toml +name = "nixl-perftest-test" + +[[Tests]] +id = "perftest.1" +num_nodes = 2 +time_limit = "00:20:00" + +name = "my_nixl_perftest_test" +description = "Example NIXL Perftest test" +test_template_name = "NixlPerftest" + + [Tests.cmd_args] + docker_image_url = "" + subtest = "sequential-ct-perftest" + num_user_requests = 100 + batch_size = 1 + num_prefill_nodes = 1 + num_decode_nodes = 1 + prefill_tp = 8 + decode_tp = 8 + model = "deepseek-r1-distill-llama-70b" + + [Tests.extra_env_vars] + CUDA_VISIBLE_DEVICES = "$SLURM_LOCALID" +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/nixl_perftest.rst b/doc/workloads/nixl_perftest.rst deleted file mode 100644 index b8e158a98..000000000 --- a/doc/workloads/nixl_perftest.rst +++ /dev/null @@ -1,86 +0,0 @@ -NIXL CTPerf -=========== - -This workload (`test_template_name` is ``NixlPerftest``) runs NIXL performance testing suite for comprehensive network performance evaluation. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_nixl_perftest_test" - description = "Example NIXL Perftest test" - test_template_name = "NixlPerftest" - - [cmd_args] - docker_image_url = "" - subtest = "sequential-ct-perftest" - num_user_requests = 1 - batch_size = 1 - num_prefill_nodes = 1 - num_decode_nodes = 1 - prefill_tp = 4 - decode_tp = 4 - isl_mean = 10000 - isl_scale = 3000 - model = "deepseek-r1-distill-llama-70b" - -Test Scenario example: - -.. code-block:: toml - - name = "nixl-perftest-test" - - [[Tests]] - id = "perftest.1" - num_nodes = 2 - time_limit = "00:20:00" - - test_name = "my_nixl_perftest_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "nixl-perftest-test" - - [[Tests]] - id = "perftest.1" - num_nodes = 2 - time_limit = "00:20:00" - - name = "my_nixl_perftest_test" - description = "Example NIXL Perftest test" - test_template_name = "NixlPerftest" - - [Tests.cmd_args] - docker_image_url = "" - subtest = "sequential-ct-perftest" - num_user_requests = 100 - batch_size = 1 - num_prefill_nodes = 1 - num_decode_nodes = 1 - prefill_tp = 8 - decode_tp = 8 - model = "deepseek-r1-distill-llama-70b" - - [Tests.extra_env_vars] - CUDA_VISIBLE_DEVICES = "$SLURM_LOCALID" - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestCmdArgs - :members: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/osu.mdx b/doc/workloads/osu.mdx new file mode 100644 index 000000000..34f64a02d --- /dev/null +++ b/doc/workloads/osu.mdx @@ -0,0 +1,78 @@ + + +This workload (`test_template_name` is `OSUBench`) allows you to execute OSU Micro Benchmarks within the CloudAI framework. + + + +## Usage Example + +Test example: + +``` toml +name = "osu_example" +test_template_name = "OSUBench" +description = "OSU Benchmark example" + +[cmd_args] +"docker_image_url" = "" +"benchmarks_dir" = "/directory/with/osu/binaries/in/container" +"benchmark" = ["osu_allreduce", "osu_allgather"] +"iterations" = 10 +"message_size" = "1024" +``` + +Test Scenario example: + +``` toml +name = "osu_example" + +[[Tests]] +id = "Tests.1" +test_name = "osu_example" +num_nodes = "2" +time_limit = "00:20:00" +``` + +Test-in-Scenario example: + +``` toml +name = "osu-test" + +[[Tests]] +id = "Tests.osu_allreduce" +num_nodes = 2 +time_limit = "00:05:00" + +name = "osu_example" +description = "OSU allreduce 1KB" +test_template_name = "OSUBench" + + [Tests.cmd_args] + docker_image_url = "" + benchmarks_dir = "/directory/with/osu/binaries/in/container" + benchmark = "osu_allreduce" + iterations = 10 + message_size = "1024" +``` + + + +## Comparison Reports + +The OSU comparison v2 report labels the X axis with the measured message sizes in bytes. Sizes are displayed at equally spaced positions, including zero-byte messages, rather than using default logarithmic ticks. When runs measure different size ranges, charts and tables use the sorted union of measured sizes. Missing measurements appear as chart gaps and `n/a` table cells. + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/osu.rst b/doc/workloads/osu.rst deleted file mode 100644 index 5992c98c6..000000000 --- a/doc/workloads/osu.rst +++ /dev/null @@ -1,84 +0,0 @@ -OSU -=== - -This workload (``test_template_name`` is ``OSUBench``) allows you to execute OSU Micro Benchmarks -within the CloudAI framework. - -Usage Example -------------- - -Test example: - -.. code-block:: toml - - name = "osu_example" - test_template_name = "OSUBench" - description = "OSU Benchmark example" - - [cmd_args] - "docker_image_url" = "" - "benchmarks_dir" = "/directory/with/osu/binaries/in/container" - "benchmark" = ["osu_allreduce", "osu_allgather"] - "iterations" = 10 - "message_size" = "1024" - -Test Scenario example: - -.. code-block:: toml - - name = "osu_example" - - [[Tests]] - id = "Tests.1" - test_name = "osu_example" - num_nodes = "2" - time_limit = "00:20:00" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "osu-test" - - [[Tests]] - id = "Tests.osu_allreduce" - num_nodes = 2 - time_limit = "00:05:00" - - name = "osu_example" - description = "OSU allreduce 1KB" - test_template_name = "OSUBench" - - [Tests.cmd_args] - docker_image_url = "" - benchmarks_dir = "/directory/with/osu/binaries/in/container" - benchmark = "osu_allreduce" - iterations = 10 - message_size = "1024" - -Comparison Reports ------------------- - -The OSU comparison v2 report labels the X axis with the measured message sizes -in bytes. Sizes are displayed at equally spaced positions, including zero-byte -messages, rather than using default logarithmic ticks. -When runs measure different size ranges, charts and tables use the sorted union -of measured sizes. Missing measurements appear as chart gaps and ``n/a`` table -cells. - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.osu_bench.osu_bench.OSUBenchCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.osu_bench.osu_bench.OSUBenchTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/sglang.mdx b/doc/workloads/sglang.mdx new file mode 100644 index 000000000..c1e4897bb --- /dev/null +++ b/doc/workloads/sglang.mdx @@ -0,0 +1,267 @@ + + +This workload (`test_template_name` is `sglang`) allows users to execute SGLang benchmarks within the CloudAI framework. + +SGLang is a high-throughput and memory-efficient inference engine for LLMs. This workload supports both aggregated and disaggregated prefill/decode modes. + + + +## Usage Examples + + + +### Test and Scenario Examples + +**test.toml (test definition)** + +``` toml +name = "sglang_test" +description = "Example SGLang benchmark" +test_template_name = "sglang" + +[cmd_args] +docker_image_url = "lmsysorg/sglang:dev-cu13" +model = "Qwen/Qwen3-8B" + +[bench_cmd_args] +random_input = 16 +random_output = 128 +max_concurrency = 16 +num_prompts = 30 + +[semantic_eval_cmd_args] +entrypoint = "python3 -m sglang.test.run_eval" +cli = "--host {host} --port {port} --eval-name gsm8k --num-examples 200 --num-threads 128 --model {model}" +``` + +**scenario.toml (scenario with one test)** + +``` toml +name = "sglang-benchmark" + +[[Tests]] +id = "sglang.1" +num_nodes = 1 +time_limit = "00:10:00" +test_name = "sglang_test" +``` + + + +### Test-in-Scenario example + +**scenario.toml (separate test toml is not needed)** + +``` toml +name = "sglang-benchmark" + +[[Tests]] +id = "sglang.1" +num_nodes = 1 +time_limit = "00:10:00" + +name = "sglang_test" +description = "Example SGLang benchmark" +test_template_name = "sglang" + +[Tests.cmd_args] +docker_image_url = "lmsysorg/sglang:dev-cu13" +model = "Qwen/Qwen3-8B" +``` + + + +## Local Models + +Set `cmd_args.model` to an absolute, container-visible path to load an existing model directory instead of downloading a model from Hugging Face. Use `extra_container_mounts` to mount a host or shared-filesystem directory. + +**test.toml (local model)** + +``` toml +name = "sglang_local_model" +description = "SGLang with a local model" +test_template_name = "sglang" +extra_container_mounts = [ + "/lustre/models/custom:/models/custom:ro", +] + +[cmd_args] +docker_image_url = "lmsysorg/sglang:dev-cu13" +model = "/models/custom" +``` + +The mounted directory must contain a model, configuration, and tokenizer supported by SGLang. + + + +## Semantic Validation + +To run GSM8K semantic validation after the serving benchmark, add `semantic_eval_cmd_args`. CloudAI reports `accuracy` from the eval output, but does not enforce an accuracy threshold. + +**test.toml (semantic validation)** + +``` toml +[semantic_eval_cmd_args] +entrypoint = "python3 -m sglang.test.run_eval" +cli = "--host {host} --port {port} --eval-name gsm8k --num-examples 200 --num-threads 128 --model {model}" +``` + +For images that still use the legacy SGLang GSM8K runner, override the entrypoint and raw CLI: + +``` toml +[semantic_eval_cmd_args] +entrypoint = "python3 -m sglang.test.few_shot_gsm8k" +cli = "--host {host} --port {port} --num-questions 200" +``` + +The `cli` string supports `{model}`, `{host}`, `{port}`, `{url}`, `{output_path}`, and `{result_dir}` placeholders. + + + +## Reporting + +After a run completes, CloudAI parses `sglang-bench.jsonl` and prints serving latency, successful prompt count, completion rate, throughput, TPS per user, and TPS per GPU. If `semantic_eval_cmd_args` is configured, CloudAI also reports semantic validation accuracy. + +The reported metric (`default`) is throughput. Additional supported metrics are `throughput`, `tps-per-user`, `tps-per-gpu`, and `accuracy`. + +CloudAI also provides the scenario-level `sglang_comparison` report. It compares SGLang test runs in the scenario and uses `bench_cmd_args` values as comparison labels. + + + +### Structured metrics + +| Metric | Unit | Preferred | Dimensions | +|----|----|----|----| +| `request_throughput` | requests/s | Higher | None | +| `output_token_throughput` | tokens/s | Higher | None | +| `ttft` (time to first token) | ms | Lower | `statistic`: mean, median, p99 | +| `tpot` (time per output token) | ms | Lower | `statistic`: mean, median, p99 | +| `accuracy` | ratio (0–1) | Higher | None | + + + +## Readiness health checks + +Healthcheck fields: + +- `healthcheck`: aggregated server and disaggregated router endpoint, default `/v1/models`. +- `serve_healthcheck`: optional override for serve, prefill, and decode servers. + +If `serve_healthcheck` is omitted, disaggregated prefill/decode servers keep the legacy `/health` endpoint. + + + +## Control number of GPUs + +GPU selection priority, from lowest to highest: + +1. `gpus_per_node` system property (scalar value) +2. `decode.gpu_ids` command argument in non-disaggregated mode when `CUDA_VISIBLE_DEVICES` is not set +3. `CUDA_VISIBLE_DEVICES` environment variable (comma-separated list of GPU IDs) +4. `gpu_ids` command argument for both `prefill` and `decode` configurations in disaggregated mode + +In disaggregated mode, define both `prefill.gpu_ids` and `decode.gpu_ids`, or omit both. + + + +## Control disaggregation + +By default, SGLang will run without disaggregation as a single process. To enable disaggregation, one needs to set `prefill` configuration: + +**test.toml (disaggregated prefill/decode)** + +``` toml +[cmd_args] +docker_image_url = "lmsysorg/sglang:dev-cu13" +model = "Qwen/Qwen3-8B" + +[cmd_args.prefill] + +[extra_env_vars] +CUDA_VISIBLE_DEVICES = "0,1,2,3" +``` + +The config above will automatically split GPUs specified in `CUDA_VISIBLE_DEVICES` into two halves, first half will be used for prefill and second half will be used for decode. + +For more control, one can specify the GPU IDs explicitly in `prefill` and `decode` configurations: + +**test.toml (disaggregated prefill/decode)** + +``` toml +[cmd_args.prefill] +gpu_ids = "0,1" + +[cmd_args.decode] +gpu_ids = "2,3" +``` + +In this case `CUDA_VISIBLE_DEVICES` will be ignored and only the GPUs specified in `gpu_ids` will be used. + + + +## Multi-node serving + +For non-disaggregated `num_nodes > 1`, CloudAI starts one `sglang.launch_server` task per serving node with shared `--dist-init-addr`, `--nnodes`, and `--node-rank "$SLURM_PROCID"`. + +For disaggregated serving over more than two nodes, set explicit role sizes: + +- `prefill.num_nodes + decode.num_nodes` must equal the test `num_nodes`. +- CloudAI assigns contiguous node slices: prefill first, decode second. +- `tp` is total per role, not per node. +- `CUDA_VISIBLE_DEVICES` and `gpu_ids` are local GPU IDs on each serving node. + +Example: four prefill nodes and four decode nodes, each with four visible GPUs: + +**scenario.toml (multi-node disaggregated serving)** + +``` toml +[[Tests]] +id = "sglang.pd_multi_node" +num_nodes = 8 +test_template_name = "sglang" + +[Tests.cmd_args.prefill] +num_nodes = 4 +tp = 16 + +[Tests.cmd_args.decode] +num_nodes = 4 +tp = 16 + +[Tests.extra_env_vars] +CUDA_VISIBLE_DEVICES = "0,1,2,3" +``` + + + +## API Documentation + + + +### SGLang Serve Arguments + + + + + +### Command Arguments + + + + + +### Benchmark Command Arguments + + + + + +### Semantic Eval Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/sglang.rst b/doc/workloads/sglang.rst deleted file mode 100644 index 185ac3030..000000000 --- a/doc/workloads/sglang.rst +++ /dev/null @@ -1,260 +0,0 @@ -SGLang -====== - -This workload (``test_template_name`` is ``sglang``) allows users to execute SGLang benchmarks within the CloudAI framework. - -SGLang is a high-throughput and memory-efficient inference engine for LLMs. This workload supports both aggregated and disaggregated prefill/decode modes. - -Usage Examples --------------- - -Test and Scenario Examples -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: toml - :caption: test.toml (test definition) - - name = "sglang_test" - description = "Example SGLang benchmark" - test_template_name = "sglang" - - [cmd_args] - docker_image_url = "lmsysorg/sglang:dev-cu13" - model = "Qwen/Qwen3-8B" - - [bench_cmd_args] - random_input = 16 - random_output = 128 - max_concurrency = 16 - num_prompts = 30 - - [semantic_eval_cmd_args] - entrypoint = "python3 -m sglang.test.run_eval" - cli = "--host {host} --port {port} --eval-name gsm8k --num-examples 200 --num-threads 128 --model {model}" - - -.. code-block:: toml - :caption: scenario.toml (scenario with one test) - - name = "sglang-benchmark" - - [[Tests]] - id = "sglang.1" - num_nodes = 1 - time_limit = "00:10:00" - test_name = "sglang_test" - -Test-in-Scenario example -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: toml - :caption: scenario.toml (separate test toml is not needed) - - name = "sglang-benchmark" - - [[Tests]] - id = "sglang.1" - num_nodes = 1 - time_limit = "00:10:00" - - name = "sglang_test" - description = "Example SGLang benchmark" - test_template_name = "sglang" - - [Tests.cmd_args] - docker_image_url = "lmsysorg/sglang:dev-cu13" - model = "Qwen/Qwen3-8B" - -Local Models ------------- -Set ``cmd_args.model`` to an absolute, container-visible path to load an existing model directory instead of -downloading a model from Hugging Face. Use ``extra_container_mounts`` to mount a host or shared-filesystem directory. - -.. code-block:: toml - :caption: test.toml (local model) - - name = "sglang_local_model" - description = "SGLang with a local model" - test_template_name = "sglang" - extra_container_mounts = [ - "/lustre/models/custom:/models/custom:ro", - ] - - [cmd_args] - docker_image_url = "lmsysorg/sglang:dev-cu13" - model = "/models/custom" - -The mounted directory must contain a model, configuration, and tokenizer supported by SGLang. - - -Semantic Validation -------------------- -To run GSM8K semantic validation after the serving benchmark, add ``semantic_eval_cmd_args``. CloudAI reports -``accuracy`` from the eval output, but does not enforce an accuracy threshold. - -.. code-block:: toml - :caption: test.toml (semantic validation) - - [semantic_eval_cmd_args] - entrypoint = "python3 -m sglang.test.run_eval" - cli = "--host {host} --port {port} --eval-name gsm8k --num-examples 200 --num-threads 128 --model {model}" - -For images that still use the legacy SGLang GSM8K runner, override the entrypoint and raw CLI: - -.. code-block:: toml - - [semantic_eval_cmd_args] - entrypoint = "python3 -m sglang.test.few_shot_gsm8k" - cli = "--host {host} --port {port} --num-questions 200" - -The ``cli`` string supports ``{model}``, ``{host}``, ``{port}``, ``{url}``, ``{output_path}``, and ``{result_dir}`` -placeholders. - - -Reporting ---------- -After a run completes, CloudAI parses ``sglang-bench.jsonl`` and prints serving latency, successful prompt count, -completion rate, throughput, TPS per user, and TPS per GPU. If ``semantic_eval_cmd_args`` is configured, CloudAI also -reports semantic validation accuracy. - -The reported metric (``default``) is throughput. Additional supported metrics are ``throughput``, ``tps-per-user``, -``tps-per-gpu``, and ``accuracy``. - -CloudAI also provides the scenario-level ``sglang_comparison`` report. It compares SGLang test runs in the scenario and -uses ``bench_cmd_args`` values as comparison labels. - -Structured metrics -~~~~~~~~~~~~~~~~~~ - -.. csv-table:: - :header: "Metric", "Unit", "Preferred", "Dimensions" - - "``request_throughput``", "requests/s", "Higher", "None" - "``output_token_throughput``", "tokens/s", "Higher", "None" - "``ttft`` (time to first token)", "ms", "Lower", "``statistic``: mean, median, p99" - "``tpot`` (time per output token)", "ms", "Lower", "``statistic``: mean, median, p99" - "``accuracy``", "ratio (0–1)", "Higher", "None" - - -Readiness health checks ------------------------ -Healthcheck fields: - -- ``healthcheck``: aggregated server and disaggregated router endpoint, default ``/v1/models``. -- ``serve_healthcheck``: optional override for serve, prefill, and decode servers. - -If ``serve_healthcheck`` is omitted, disaggregated prefill/decode servers keep the legacy ``/health`` endpoint. - - -Control number of GPUs ----------------------- -GPU selection priority, from lowest to highest: - -1. ``gpus_per_node`` system property (scalar value) -2. ``decode.gpu_ids`` command argument in non-disaggregated mode when ``CUDA_VISIBLE_DEVICES`` is not set -3. ``CUDA_VISIBLE_DEVICES`` environment variable (comma-separated list of GPU IDs) -4. ``gpu_ids`` command argument for both ``prefill`` and ``decode`` configurations in disaggregated mode - -In disaggregated mode, define both ``prefill.gpu_ids`` and ``decode.gpu_ids``, or omit both. - - -Control disaggregation ----------------------- -By default, SGLang will run without disaggregation as a single process. To enable disaggregation, one needs to set ``prefill`` configuration: - -.. code-block:: toml - :caption: test.toml (disaggregated prefill/decode) - - [cmd_args] - docker_image_url = "lmsysorg/sglang:dev-cu13" - model = "Qwen/Qwen3-8B" - - [cmd_args.prefill] - - [extra_env_vars] - CUDA_VISIBLE_DEVICES = "0,1,2,3" - -The config above will automatically split GPUs specified in ``CUDA_VISIBLE_DEVICES`` into two halves, first half will be used for prefill and second half will be used for decode. - -For more control, one can specify the GPU IDs explicitly in ``prefill`` and ``decode`` configurations: - -.. code-block:: toml - :caption: test.toml (disaggregated prefill/decode) - - [cmd_args.prefill] - gpu_ids = "0,1" - - [cmd_args.decode] - gpu_ids = "2,3" - -In this case ``CUDA_VISIBLE_DEVICES`` will be ignored and only the GPUs specified in ``gpu_ids`` will be used. - -Multi-node serving ------------------- -For non-disaggregated ``num_nodes > 1``, CloudAI starts one ``sglang.launch_server`` task per serving node with shared -``--dist-init-addr``, ``--nnodes``, and ``--node-rank "$SLURM_PROCID"``. - -For disaggregated serving over more than two nodes, set explicit role sizes: - -- ``prefill.num_nodes + decode.num_nodes`` must equal the test ``num_nodes``. -- CloudAI assigns contiguous node slices: prefill first, decode second. -- ``tp`` is total per role, not per node. -- ``CUDA_VISIBLE_DEVICES`` and ``gpu_ids`` are local GPU IDs on each serving node. - -Example: four prefill nodes and four decode nodes, each with four visible GPUs: - -.. code-block:: toml - :caption: scenario.toml (multi-node disaggregated serving) - - [[Tests]] - id = "sglang.pd_multi_node" - num_nodes = 8 - test_template_name = "sglang" - - [Tests.cmd_args.prefill] - num_nodes = 4 - tp = 16 - - [Tests.cmd_args.decode] - num_nodes = 4 - tp = 16 - - [Tests.extra_env_vars] - CUDA_VISIBLE_DEVICES = "0,1,2,3" - -API Documentation ------------------ - -SGLang Serve Arguments -~~~~~~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.sglang.sglang.SglangArgs - :members: - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.sglang.sglang.SglangCmdArgs - :members: - :show-inheritance: - -Benchmark Command Arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.sglang.sglang.SglangBenchCmdArgs - :members: - :show-inheritance: - -Semantic Eval Command Arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.sglang.sglang.SglangSemanticEvalCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.sglang.sglang.SglangTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/sleep.mdx b/doc/workloads/sleep.mdx new file mode 100644 index 000000000..f8399068a --- /dev/null +++ b/doc/workloads/sleep.mdx @@ -0,0 +1,65 @@ + + +This workload (`test_template_name` is `Sleep`) executes a simple sleep command for testing and timing purposes. Useful for testing schedulers and system behavior. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_sleep_test" +description = "Example Sleep test" +test_template_name = "Sleep" + +[cmd_args] +seconds = 30 +``` + +Test Scenario example: + +``` toml +name = "sleep-test" + +[[Tests]] +id = "sleep.1" +num_nodes = 1 +time_limit = "00:02:00" + +test_name = "my_sleep_test" +``` + +Test-in-Scenario example: + +``` toml +name = "sleep-test" + +[[Tests]] +id = "sleep.1" +num_nodes = 1 +time_limit = "00:02:00" + +name = "my_sleep_test" +description = "Example Sleep test" +test_template_name = "Sleep" + + [Tests.cmd_args] + seconds = 30 +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/sleep.rst b/doc/workloads/sleep.rst deleted file mode 100644 index 1cfbd67b8..000000000 --- a/doc/workloads/sleep.rst +++ /dev/null @@ -1,66 +0,0 @@ -Sleep -===== - -This workload (`test_template_name` is ``Sleep``) executes a simple sleep command for testing and timing purposes. Useful for testing schedulers and system behavior. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_sleep_test" - description = "Example Sleep test" - test_template_name = "Sleep" - - [cmd_args] - seconds = 30 - -Test Scenario example: - -.. code-block:: toml - - name = "sleep-test" - - [[Tests]] - id = "sleep.1" - num_nodes = 1 - time_limit = "00:02:00" - - test_name = "my_sleep_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "sleep-test" - - [[Tests]] - id = "sleep.1" - num_nodes = 1 - time_limit = "00:02:00" - - name = "my_sleep_test" - description = "Example Sleep test" - test_template_name = "Sleep" - - [Tests.cmd_args] - seconds = 30 - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.sleep.sleep.SleepCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.sleep.sleep.SleepTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/slurm_container.mdx b/doc/workloads/slurm_container.mdx new file mode 100644 index 000000000..a5500aab2 --- /dev/null +++ b/doc/workloads/slurm_container.mdx @@ -0,0 +1,75 @@ + + +This workload (`test_template_name` is `SlurmContainer`) executes containerized workloads using Slurm with custom container configurations. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "my_container_test" +description = "Example Slurm container test" +test_template_name = "SlurmContainer" + +[cmd_args] +check_exit_code = true +docker_image_url = "/path/to/container.sqsh" +cmd = "python train.py" +``` + +Test Scenario example: + +``` toml +name = "slurm-container-test" + +[[Tests]] +id = "container.1" +num_nodes = 2 +time_limit = "01:00:00" + +test_name = "my_container_test" +``` + +Test-in-Scenario example: + +``` toml +name = "slurm-container-test" + +[[Tests]] +id = "container.1" +num_nodes = 2 +time_limit = "01:00:00" + +name = "my_container_test" +description = "Example Slurm container test" +test_template_name = "SlurmContainer" + + [Tests.cmd_args] + check_exit_code = true + docker_image_url = "/path/to/container.sqsh" + cmd = "python train.py" +``` + + + +## Run Status + +Exit-code checking is disabled by default for backwards compatibility. Set `check_exit_code = true` in `cmd_args` to record the aggregate `srun` exit code in the per-test `exit_code.txt` file and use it to determine success. Only integer `0` is successful. Any non-zero integer, or a missing, malformed, unreadable, or undecodable file, marks the run as failed. Slurm-style exit code `0:0` is not accepted. + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/slurm_container.rst b/doc/workloads/slurm_container.rst deleted file mode 100644 index e745f5cf5..000000000 --- a/doc/workloads/slurm_container.rst +++ /dev/null @@ -1,80 +0,0 @@ -Slurm Container -=============== - -This workload (`test_template_name` is ``SlurmContainer``) executes containerized workloads using Slurm with custom container configurations. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "my_container_test" - description = "Example Slurm container test" - test_template_name = "SlurmContainer" - - [cmd_args] - check_exit_code = true - docker_image_url = "/path/to/container.sqsh" - cmd = "python train.py" - -Test Scenario example: - -.. code-block:: toml - - name = "slurm-container-test" - - [[Tests]] - id = "container.1" - num_nodes = 2 - time_limit = "01:00:00" - - test_name = "my_container_test" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "slurm-container-test" - - [[Tests]] - id = "container.1" - num_nodes = 2 - time_limit = "01:00:00" - - name = "my_container_test" - description = "Example Slurm container test" - test_template_name = "SlurmContainer" - - [Tests.cmd_args] - check_exit_code = true - docker_image_url = "/path/to/container.sqsh" - cmd = "python train.py" - -Run Status ----------- - -Exit-code checking is disabled by default for backwards compatibility. Set -``check_exit_code = true`` in ``cmd_args`` to record the aggregate ``srun`` exit -code in the per-test ``exit_code.txt`` file and use it to determine success. -Only integer ``0`` is successful. Any non-zero integer, or a missing, malformed, -unreadable, or undecodable file, marks the run as failed. Slurm-style exit code -``0:0`` is not accepted. - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.slurm_container.slurm_container.SlurmContainerCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.slurm_container.slurm_container.SlurmContainerTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/ucc.mdx b/doc/workloads/ucc.mdx new file mode 100644 index 000000000..347311cf5 --- /dev/null +++ b/doc/workloads/ucc.mdx @@ -0,0 +1,65 @@ + + +This workload (`test_template_name` is `UCCTest`) allows users to execute UCC benchmarks within the CloudAI framework. + + + +## Usage Examples + +Test TOML example: + +``` toml +name = "ucc" +description = "Example UCC test" +test_template_name = "UCCTest" + +[cmd_args] +docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" +``` + +Test Scenario example: + +``` toml +name = "ucc-test" + +[[Tests]] +id = "ucc.1" +num_nodes = 1 +time_limit = "00:02:00" + +test_name = "ucc" +``` + +Test-in-Scenario example: + +``` toml +name = "ucc-test" + +[[Tests]] +id = "ucc.1" +num_nodes = 1 +time_limit = "00:02:00" + +name = "ucc" +description = "Example UCC test" +test_template_name = "UCCTest" + +[Tests.cmd_args] +docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" +``` + + + +## API Documentation + + + +### Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/ucc.rst b/doc/workloads/ucc.rst deleted file mode 100644 index 2d559ddc6..000000000 --- a/doc/workloads/ucc.rst +++ /dev/null @@ -1,66 +0,0 @@ -UCC -=== - -This workload (`test_template_name` is ``UCCTest``) allows users to execute UCC benchmarks within the CloudAI framework. - -Usage Examples --------------- - -Test TOML example: - -.. code-block:: toml - - name = "ucc" - description = "Example UCC test" - test_template_name = "UCCTest" - - [cmd_args] - docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" - -Test Scenario example: - -.. code-block:: toml - - name = "ucc-test" - - [[Tests]] - id = "ucc.1" - num_nodes = 1 - time_limit = "00:02:00" - - test_name = "ucc" - -Test-in-Scenario example: - -.. code-block:: toml - - name = "ucc-test" - - [[Tests]] - id = "ucc.1" - num_nodes = 1 - time_limit = "00:02:00" - - name = "ucc" - description = "Example UCC test" - test_template_name = "UCCTest" - - [Tests.cmd_args] - docker_image_url = "nvcr.io#nvidia/pytorch:25.06-py3" - -API Documentation ------------------ - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.ucc_test.ucc.UCCCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.ucc_test.ucc.UCCTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/vllm.mdx b/doc/workloads/vllm.mdx new file mode 100644 index 000000000..1777b3def --- /dev/null +++ b/doc/workloads/vllm.mdx @@ -0,0 +1,285 @@ + + +vLLM workload (`test_template_name` is `vllm`) allows users to execute vLLM benchmarks within the CloudAI framework. + +vLLM is a high-throughput and memory-efficient inference engine for LLMs. This workload supports both aggregated and disaggregated prefill/decode modes. + + + +## Usage Examples + + + +### Test and Scenario Examples + +**test.toml (test definition)** + +``` toml +name = "vllm_test" +description = "Example vLLM test" +test_template_name = "vllm" + +[cmd_args] +docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" +model = "Qwen/Qwen3-0.6B" + +[bench_cmd_args] +random_input_len = 16 +random_output_len = 128 +max_concurrency = 16 +num_prompts = 30 + +[semantic_eval_cmd_args] +entrypoint = "python3 /opt/vllm/tests/evals/gsm8k/gsm8k_eval.py" +cli = "--host {host} --port {port} --num-questions 200 --save-results {output_path}/vllm-gsm8k.json" +``` + +**scenario.toml (scenario with one test)** + +``` toml +name = "vllm-benchmark" + +[[Tests]] +id = "vllm.1" +num_nodes = 1 +time_limit = "00:10:00" +test_name = "vllm_test" +``` + + + +### Test-in-Scenario example + +**scenario.toml (separate test toml is not needed)** + +``` toml +name = "vllm-benchmark" + +[[Tests]] +id = "vllm.1" +num_nodes = 1 +time_limit = "00:10:00" + +name = "vllm_test" +description = "Example vLLM test" +test_template_name = "vllm" + +[Tests.cmd_args] +docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" +model = "Qwen/Qwen3-0.6B" +``` + + + +## Local Models + +Set `cmd_args.model` to an absolute, container-visible path to load an existing model directory instead of downloading a model from Hugging Face. Use `extra_container_mounts` to mount a host or shared-filesystem directory. + +**test.toml (local model)** + +``` toml +name = "vllm_local_model" +description = "vLLM with a local model" +test_template_name = "vllm" +extra_container_mounts = [ + "/lustre/models/custom:/models/custom:ro", +] + +[cmd_args] +docker_image_url = "vllm/vllm-openai:v0.14.0-cu130" +model = "/models/custom" +``` + +The mounted directory must contain a model, configuration, and tokenizer supported by vLLM. + + + +## Semantic Validation + +To run GSM8K semantic validation after the serving benchmark, add `semantic_eval_cmd_args`. CloudAI reports `accuracy` from the eval output, but does not enforce an accuracy threshold. + +**test.toml (semantic validation)** + +``` toml +[semantic_eval_cmd_args] +entrypoint = "python3 /opt/vllm/tests/evals/gsm8k/gsm8k_eval.py" +cli = "--host {host} --port {port} --num-questions 200 --save-results {output_path}/vllm-gsm8k.json" +``` + +If the runtime image does not contain the eval script, mount a vLLM repository with existing `git_repos` support and point `entrypoint` at the mounted path. + +The `cli` string supports `{model}`, `{host}`, `{port}`, `{url}`, `{output_path}`, and `{result_dir}` placeholders. + + + +## Reporting + +After a run completes, CloudAI parses `vllm-bench.json` and prints serving latency, successful prompt count, completion rate, throughput, TPS per user, and TPS per GPU. If `semantic_eval_cmd_args` is configured, CloudAI also reports semantic validation accuracy. + +The reported metric (`default`) is throughput. Additional supported metrics are `throughput`, `tps-per-user`, `tps-per-gpu`, and `accuracy`. + +CloudAI also provides the scenario-level `vllm_comparison` report. It compares vLLM test runs in the scenario and uses `bench_cmd_args` values as comparison labels. + + + +### Structured metrics + +| Metric | Unit | Preferred | Dimensions | +|----|----|----|----| +| `request_throughput` | requests/s | Higher | None | +| `output_token_throughput` | tokens/s | Higher | None | +| `ttft` (time to first token) | ms | Lower | `statistic`: mean, median, p99 | +| `tpot` (time per output token) | ms | Lower | `statistic`: mean, median, p99 | +| `accuracy` | ratio (0–1) | Higher | None | + + + +## Controlling the Number of GPUs + +GPU selection priority, from lowest to highest: + +1. `gpus_per_node` system property (scalar value) +2. `decode.gpu_ids` command argument in non-disaggregated mode when `CUDA_VISIBLE_DEVICES` is not set +3. `CUDA_VISIBLE_DEVICES` environment variable (comma-separated list of GPU IDs) +4. `gpu_ids` command argument for both `prefill` and `decode` configurations in disaggregated mode + +In disaggregated mode, define both `prefill.gpu_ids` and `decode.gpu_ids`, or omit both. + + + +## Controlling Disaggregation + +By default, vLLM will run without disaggregation as a single process. To enable disaggregation, one needs to set `prefill` configuration: + +**test.toml (disaggregated prefill/decode)** + +``` toml +[cmd_args] +docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" +model = "Qwen/Qwen3-0.6B" + +[cmd_args.prefill] + +[extra_env_vars] +CUDA_VISIBLE_DEVICES = "0,1,2,3" +``` + +The config above, will automatically split GPUs specified in `CUDA_VISIBLE_DEVICES` into two: - The first half will be used for prefill - The second half will be used for decode + +For more control, users can specify the GPU IDs explicitly in `prefill` and `decode` configurations: + +**test.toml (disaggregated prefill/decode)** + +``` toml +[cmd_args.prefill] +gpu_ids = "0,1" + +[cmd_args.decode] +gpu_ids = "2,3" +``` + +In this case `CUDA_VISIBLE_DEVICES` will be ignored and only the GPUs specified in `gpu_ids` will be used. + + + +## Multi-node serving + +For non-disaggregated `num_nodes > 1`, CloudAI creates one Ray cluster and starts `vllm serve` on the head node with `--distributed-executor-backend ray`. + +For disaggregated serving over more than two nodes, set explicit role sizes: + +- `prefill.num_nodes + decode.num_nodes` must equal the test `num_nodes`. +- CloudAI assigns contiguous node slices: prefill first, decode second. +- `tensor_parallel_size` is total per role, not per node. +- `CUDA_VISIBLE_DEVICES` and `gpu_ids` are local GPU IDs on each serving node. + +Example: four prefill nodes and four decode nodes, each with four visible GPUs: + +**scenario.toml (multi-node disaggregated serving)** + +``` toml +[[Tests]] +id = "vllm.pd_multi_node" +num_nodes = 8 +test_template_name = "vllm" + +[Tests.cmd_args.prefill] +num_nodes = 4 +tensor_parallel_size = 16 + +[Tests.cmd_args.decode] +num_nodes = 4 +tensor_parallel_size = 16 + +[Tests.extra_env_vars] +CUDA_VISIBLE_DEVICES = "0,1,2,3" +``` + + + +## Readiness health checks + +Healthcheck fields: + +- `healthcheck`: aggregated server endpoint, default `/healthcheck`. +- `serve_healthcheck`: optional override for serve, prefill, and decode servers. +- `proxy_healthcheck`: disaggregated proxy/router endpoint, default `/healthcheck`. + +If `serve_healthcheck` is omitted, disaggregated prefill/decode servers keep the legacy `/health` endpoint. If a disaggregated config sets `healthcheck` but omits `proxy_healthcheck`, the proxy/router uses `healthcheck` for backward compatibility. + + + +## Controlling `proxy_script` + +`proxy_script` is used to proxy the requests from the client to the prefill and decode instances. It is ignored for non-disaggregated mode. Default value can be found below. + +It can be overridden by setting `proxy_script` by using the latest version of the script from vLLM repository: + +**test_scenario.toml (override proxy_script)** + +``` toml +[[Tests.git_repos]] +url = "https://github.com/vllm-project/vllm.git" +commit = "main" +mount_as = "/vllm_repo" + +[Tests.cmd_args] +docker_image_url = "vllm/vllm-openai:v0.14.0-cu130" +proxy_script = "/vllm_repo/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" +``` + +In this case the proxy script will be mounted from the vLLM repository (cloned locally) as `/vllm_repo` and used for the test. + + + +## API Documentation + + + +### vLLM Serve Arguments + + + + + +### Command Arguments + + + + + +### Benchmark Command Arguments + + + + + +### Semantic Eval Command Arguments + + + + + +### Test Definition + + diff --git a/doc/workloads/vllm.rst b/doc/workloads/vllm.rst deleted file mode 100644 index c1cc86b6b..000000000 --- a/doc/workloads/vllm.rst +++ /dev/null @@ -1,282 +0,0 @@ -vLLM -==== - -vLLM workload (``test_template_name`` is ``vllm``) allows users to execute vLLM benchmarks within the CloudAI framework. - -vLLM is a high-throughput and memory-efficient inference engine for LLMs. This workload supports both aggregated and disaggregated prefill/decode modes. - -Usage Examples --------------- - -Test and Scenario Examples -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: toml - :caption: test.toml (test definition) - - name = "vllm_test" - description = "Example vLLM test" - test_template_name = "vllm" - - [cmd_args] - docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" - model = "Qwen/Qwen3-0.6B" - - [bench_cmd_args] - random_input_len = 16 - random_output_len = 128 - max_concurrency = 16 - num_prompts = 30 - - [semantic_eval_cmd_args] - entrypoint = "python3 /opt/vllm/tests/evals/gsm8k/gsm8k_eval.py" - cli = "--host {host} --port {port} --num-questions 200 --save-results {output_path}/vllm-gsm8k.json" - - -.. code-block:: toml - :caption: scenario.toml (scenario with one test) - - name = "vllm-benchmark" - - [[Tests]] - id = "vllm.1" - num_nodes = 1 - time_limit = "00:10:00" - test_name = "vllm_test" - -Test-in-Scenario example -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: toml - :caption: scenario.toml (separate test toml is not needed) - - name = "vllm-benchmark" - - [[Tests]] - id = "vllm.1" - num_nodes = 1 - time_limit = "00:10:00" - - name = "vllm_test" - description = "Example vLLM test" - test_template_name = "vllm" - - [Tests.cmd_args] - docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" - model = "Qwen/Qwen3-0.6B" - -Local Models ------------- -Set ``cmd_args.model`` to an absolute, container-visible path to load an existing model directory instead of -downloading a model from Hugging Face. Use ``extra_container_mounts`` to mount a host or shared-filesystem directory. - -.. code-block:: toml - :caption: test.toml (local model) - - name = "vllm_local_model" - description = "vLLM with a local model" - test_template_name = "vllm" - extra_container_mounts = [ - "/lustre/models/custom:/models/custom:ro", - ] - - [cmd_args] - docker_image_url = "vllm/vllm-openai:v0.14.0-cu130" - model = "/models/custom" - -The mounted directory must contain a model, configuration, and tokenizer supported by vLLM. - - -Semantic Validation -------------------- -To run GSM8K semantic validation after the serving benchmark, add ``semantic_eval_cmd_args``. CloudAI reports -``accuracy`` from the eval output, but does not enforce an accuracy threshold. - -.. code-block:: toml - :caption: test.toml (semantic validation) - - [semantic_eval_cmd_args] - entrypoint = "python3 /opt/vllm/tests/evals/gsm8k/gsm8k_eval.py" - cli = "--host {host} --port {port} --num-questions 200 --save-results {output_path}/vllm-gsm8k.json" - -If the runtime image does not contain the eval script, mount a vLLM repository with existing ``git_repos`` support and -point ``entrypoint`` at the mounted path. - -The ``cli`` string supports ``{model}``, ``{host}``, ``{port}``, ``{url}``, ``{output_path}``, and ``{result_dir}`` -placeholders. - - -Reporting ---------- -After a run completes, CloudAI parses ``vllm-bench.json`` and prints serving latency, successful prompt count, -completion rate, throughput, TPS per user, and TPS per GPU. If ``semantic_eval_cmd_args`` is configured, CloudAI also -reports semantic validation accuracy. - -The reported metric (``default``) is throughput. Additional supported metrics are ``throughput``, ``tps-per-user``, -``tps-per-gpu``, and ``accuracy``. - -CloudAI also provides the scenario-level ``vllm_comparison`` report. It compares vLLM test runs in the scenario and -uses ``bench_cmd_args`` values as comparison labels. - -Structured metrics -~~~~~~~~~~~~~~~~~~ - -.. csv-table:: - :header: "Metric", "Unit", "Preferred", "Dimensions" - - "``request_throughput``", "requests/s", "Higher", "None" - "``output_token_throughput``", "tokens/s", "Higher", "None" - "``ttft`` (time to first token)", "ms", "Lower", "``statistic``: mean, median, p99" - "``tpot`` (time per output token)", "ms", "Lower", "``statistic``: mean, median, p99" - "``accuracy``", "ratio (0–1)", "Higher", "None" - - -Controlling the Number of GPUs -------------------------------- -GPU selection priority, from lowest to highest: - -1. ``gpus_per_node`` system property (scalar value) -2. ``decode.gpu_ids`` command argument in non-disaggregated mode when ``CUDA_VISIBLE_DEVICES`` is not set -3. ``CUDA_VISIBLE_DEVICES`` environment variable (comma-separated list of GPU IDs) -4. ``gpu_ids`` command argument for both ``prefill`` and ``decode`` configurations in disaggregated mode - -In disaggregated mode, define both ``prefill.gpu_ids`` and ``decode.gpu_ids``, or omit both. - -Controlling Disaggregation --------------------------- -By default, vLLM will run without disaggregation as a single process. To enable disaggregation, one needs to set ``prefill`` configuration: - -.. code-block:: toml - :caption: test.toml (disaggregated prefill/decode) - - [cmd_args] - docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" - model = "Qwen/Qwen3-0.6B" - - [cmd_args.prefill] - - [extra_env_vars] - CUDA_VISIBLE_DEVICES = "0,1,2,3" - -The config above, will automatically split GPUs specified in ``CUDA_VISIBLE_DEVICES`` into two: -- The first half will be used for prefill -- The second half will be used for decode - -For more control, users can specify the GPU IDs explicitly in ``prefill`` and ``decode`` configurations: - -.. code-block:: toml - :caption: test.toml (disaggregated prefill/decode) - - [cmd_args.prefill] - gpu_ids = "0,1" - - [cmd_args.decode] - gpu_ids = "2,3" - -In this case ``CUDA_VISIBLE_DEVICES`` will be ignored and only the GPUs specified in ``gpu_ids`` will be used. - - -Multi-node serving ------------------- -For non-disaggregated ``num_nodes > 1``, CloudAI creates one Ray cluster and starts ``vllm serve`` on the head node with -``--distributed-executor-backend ray``. - -For disaggregated serving over more than two nodes, set explicit role sizes: - -- ``prefill.num_nodes + decode.num_nodes`` must equal the test ``num_nodes``. -- CloudAI assigns contiguous node slices: prefill first, decode second. -- ``tensor_parallel_size`` is total per role, not per node. -- ``CUDA_VISIBLE_DEVICES`` and ``gpu_ids`` are local GPU IDs on each serving node. - -Example: four prefill nodes and four decode nodes, each with four visible GPUs: - -.. code-block:: toml - :caption: scenario.toml (multi-node disaggregated serving) - - [[Tests]] - id = "vllm.pd_multi_node" - num_nodes = 8 - test_template_name = "vllm" - - [Tests.cmd_args.prefill] - num_nodes = 4 - tensor_parallel_size = 16 - - [Tests.cmd_args.decode] - num_nodes = 4 - tensor_parallel_size = 16 - - [Tests.extra_env_vars] - CUDA_VISIBLE_DEVICES = "0,1,2,3" - - -Readiness health checks ------------------------ -Healthcheck fields: - -- ``healthcheck``: aggregated server endpoint, default ``/healthcheck``. -- ``serve_healthcheck``: optional override for serve, prefill, and decode servers. -- ``proxy_healthcheck``: disaggregated proxy/router endpoint, default ``/healthcheck``. - -If ``serve_healthcheck`` is omitted, disaggregated prefill/decode servers keep the legacy ``/health`` endpoint. If a -disaggregated config sets ``healthcheck`` but omits ``proxy_healthcheck``, the proxy/router uses ``healthcheck`` for -backward compatibility. - - -Controlling ``proxy_script`` ------------------------------ -``proxy_script`` is used to proxy the requests from the client to the prefill and decode instances. It is ignored for non-disaggregated mode. Default value can be found below. - -It can be overridden by setting ``proxy_script`` by using the latest version of the script from vLLM repository: - -.. code-block:: toml - :caption: test_scenario.toml (override proxy_script) - - [[Tests.git_repos]] - url = "https://github.com/vllm-project/vllm.git" - commit = "main" - mount_as = "/vllm_repo" - - [Tests.cmd_args] - docker_image_url = "vllm/vllm-openai:v0.14.0-cu130" - proxy_script = "/vllm_repo/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" - -In this case the proxy script will be mounted from the vLLM repository (cloned locally) as ``/vllm_repo`` and used for the test. - - -API Documentation ------------------ - -vLLM Serve Arguments -~~~~~~~~~~~~~~~~~~~~ - -.. autopydantic_model:: cloudai.workloads.vllm.vllm.VllmArgs - :members: - -Command Arguments -~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.vllm.vllm.VllmCmdArgs - :members: - :show-inheritance: - -Benchmark Command Arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.vllm.vllm.VllmBenchCmdArgs - :members: - :show-inheritance: - -Semantic Eval Command Arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.vllm.vllm.VllmSemanticEvalCmdArgs - :members: - :show-inheritance: - -Test Definition -~~~~~~~~~~~~~~~ - -.. autoclass:: cloudai.workloads.vllm.vllm.VllmTestDefinition - :members: - :show-inheritance: diff --git a/doc/workloads/workloads_requirements_installation.rst b/doc/workloads/workloads_requirements_installation.mdx similarity index 61% rename from doc/workloads/workloads_requirements_installation.rst rename to doc/workloads/workloads_requirements_installation.mdx index db3b5951f..4330be4a5 100644 --- a/doc/workloads/workloads_requirements_installation.rst +++ b/doc/workloads/workloads_requirements_installation.mdx @@ -1,45 +1,48 @@ -Installation Requirements -========================== + CloudAI workloads can define multiple installables as prerequisites. The installable can be a container image, git repository, HF model, etc. + -Setting Up Access to the Private NGC Registry ---------------------------------------------- +## Setting Up Access to the Private NGC Registry First, make sure you have access to the Docker repository. Proceed as follows: -1. **Sign In**: Go to `NGC signin`_ and sign in with your credentials. -2. **Generate API Key**: +1. **Sign In**: Go to [NGC signin](https://ngc.nvidia.com/signin) and sign in with your credentials. + +2. **Generate API Key**: - On the top right corner, click on the dropdown menu next to your profile + - Select **Setup** + - In the **Setup** section, find **Keys/Secrets** + - Click **Generate API Key** and confirm when prompted. A new API key will be presented + - **Note**: Save this API key locally as you will not be able to view it again on NGC - - Set up your enroot credentials. Make sure you have the correct credentials under **~/.config/enroot/.credentials**: - .. code-block:: text + - Set up your enroot credentials. Make sure you have the correct credentials under **~/.config/enroot/.credentials**: - machine nvcr.io login $oauthtoken password + > ``` text + > machine nvcr.io login $oauthtoken password + > ``` - Replace `` with your respective credentials. Keep `$oauthtoken` as is. + -.. _NGC signin: https://ngc.nvidia.com/signin +## Hugging Face Models +Some workloads require Hugging Face models. CloudAI will download the models from Hugging Face and cache them in the location specified by System's `hf_home_path` field. By default, it is set to `/huggingface`, but any other location can be specified. When Slurm is used, this location will be mounted to the container. -Hugging Face Models -------------------- + -Some workloads require Hugging Face models. CloudAI will download the models from Hugging Face and cache them in the location specified by System's ``hf_home_path`` field. By default, it is set to ``/huggingface``, but any other location can be specified. When Slurm is used, this location will be mounted to the container. - -Authentication with Hugging Face -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### Authentication with Hugging Face As of now, CloudAI does not handle authentication with Hugging Face, so it is up to the user to enable authentication with Hugging Face in the shell where CloudAI is run. Users might need to run the following command: -.. code-block:: bash - - uv run hf auth login +``` bash +uv run hf auth login +``` Once done, all Hugging Face models will be downloaded using existing authentication. diff --git a/fern/docs.yml b/fern/docs.yml new file mode 100644 index 000000000..40671f633 --- /dev/null +++ b/fern/docs.yml @@ -0,0 +1,125 @@ +# yaml-language-server: $schema=https://schema.buildwithfern.dev/docs-yml.json +instances: +- url: ${FERN_DOCS_URL} +title: CloudAI +settings: + substitute-env-vars: true +global-theme: nvidia +navbar-links: +- type: github + value: https://github.com/NVIDIA/cloudai +navigation: +- page: CloudAI Benchmark Framework + path: ../doc/index.mdx + slug: / +- page: Tutorial + path: ../doc/Tutorial.mdx + slug: Tutorial +- page: User Guide + path: ../doc/USER_GUIDE.mdx + slug: USER_GUIDE +- page: Python API + path: ../doc/api.mdx + slug: api +- section: Reporting + skip-slug: true + contents: + - page: Reporting + path: ../doc/reporting.mdx + slug: reporting + - page: Reports + path: ../doc/reports.mdx + slug: reports + - page: Pareto Frontiers + path: ../doc/pareto-frontiers.mdx + slug: pareto-frontiers + - page: Training Report Schema + path: ../doc/training-report-schema.mdx + slug: training-report-schema +- page: Systems + path: ../doc/systems.mdx + slug: systems +- section: Workloads + skip-slug: true + contents: + - page: Workloads + path: ../doc/workloads/index.mdx + slug: workloads/index + - page: AI Dynamo + path: ../doc/workloads/ai_dynamo.mdx + slug: workloads/ai_dynamo + - page: AIConfigurator + path: ../doc/workloads/aiconfigurator.mdx + slug: workloads/aiconfigurator + - page: Bash Command + path: ../doc/workloads/bash_cmd.mdx + slug: workloads/bash_cmd + - page: Chakra Replay + path: ../doc/workloads/chakra_replay.mdx + slug: workloads/chakra_replay + - page: DDLB + path: ../doc/workloads/ddlb.mdx + slug: workloads/ddlb + - page: DeepEP Benchmark + path: ../doc/workloads/deepep.mdx + slug: workloads/deepep + - page: DynamoMocker + path: ../doc/workloads/dynamo_mocker.mdx + slug: workloads/dynamo_mocker + - page: fio + path: ../doc/workloads/fio.mdx + slug: workloads/fio + - page: JaxToolbox workloads (DEPRECATED) + path: ../doc/workloads/jax_toolbox.mdx + slug: workloads/jax_toolbox + - page: MegatronBridge + path: ../doc/workloads/megatron_bridge.mdx + slug: workloads/megatron_bridge + - page: NCCL + path: ../doc/workloads/nccl.mdx + slug: workloads/nccl + - page: NeMo v1.0 aka NemoLauncher (DEPRECATED) + path: ../doc/workloads/nemo_launcher.mdx + slug: workloads/nemo_launcher + - page: Nemo Run + path: ../doc/workloads/nemo_run.mdx + slug: workloads/nemo_run + - page: NIXL Bench + path: ../doc/workloads/nixl_bench.mdx + slug: workloads/nixl_bench + - page: NIXL EP + path: ../doc/workloads/nixl_ep.mdx + slug: workloads/nixl_ep + - page: NIXL KVBench + path: ../doc/workloads/nixl_kvbench.mdx + slug: workloads/nixl_kvbench + - page: NIXL CTPerf + path: ../doc/workloads/nixl_perftest.mdx + slug: workloads/nixl_perftest + - page: OSU + path: ../doc/workloads/osu.mdx + slug: workloads/osu + - page: SGLang + path: ../doc/workloads/sglang.mdx + slug: workloads/sglang + - page: Sleep + path: ../doc/workloads/sleep.mdx + slug: workloads/sleep + - page: Slurm Container + path: ../doc/workloads/slurm_container.mdx + slug: workloads/slurm_container + - page: UCC + path: ../doc/workloads/ucc.mdx + slug: workloads/ucc + - page: vLLM + path: ../doc/workloads/vllm.mdx + slug: workloads/vllm + - page: Installation Requirements + path: ../doc/workloads/workloads_requirements_installation.mdx + slug: workloads/workloads_requirements_installation +- page: Development + path: ../doc/DEV.mdx + slug: DEV +- page: Troubleshooting + path: ../doc/troubleshooting.mdx + slug: troubleshooting diff --git a/fern/fern.config.json b/fern/fern.config.json new file mode 100644 index 000000000..5d5e03445 --- /dev/null +++ b/fern/fern.config.json @@ -0,0 +1,4 @@ +{ + "organization": "nvidia", + "version": "5.176.0" +} diff --git a/fern/package-lock.json b/fern/package-lock.json new file mode 100644 index 000000000..b799c59b1 --- /dev/null +++ b/fern/package-lock.json @@ -0,0 +1,210 @@ +{ + "name": "cloudai-docs", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cloudai-docs", + "devDependencies": { + "fern-api": "5.176.0" + } + }, + "node_modules/@boundaryml/baml": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml/-/baml-0.219.0.tgz", + "integrity": "sha512-hE6t6G/1Td9yYN/T6E13igF06ZHD0J9dr5FH3tjCCZQplqTLBiVn4wyVoboK7ypjdHdGkg6O0vAcXrcbdwM6pA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@scarf/scarf": "^1.3.0" + }, + "bin": { + "baml": "cli.js", + "baml-cli": "cli.js" + }, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@boundaryml/baml-darwin-arm64": "0.219.0", + "@boundaryml/baml-darwin-x64": "0.219.0", + "@boundaryml/baml-linux-arm64-gnu": "0.219.0", + "@boundaryml/baml-linux-arm64-musl": "0.219.0", + "@boundaryml/baml-linux-x64-gnu": "0.219.0", + "@boundaryml/baml-linux-x64-musl": "0.219.0", + "@boundaryml/baml-win32-arm64-msvc": "0.219.0", + "@boundaryml/baml-win32-x64-msvc": "0.219.0" + } + }, + "node_modules/@boundaryml/baml-darwin-arm64": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-darwin-arm64/-/baml-darwin-arm64-0.219.0.tgz", + "integrity": "sha512-jhPN83UM+9Y99aCbzIS/3OpC7N8dOukPZO2piueIbzsmpaJrzTW+srKxYtVDE26Q9riFypOmizSs5SBFst04Fw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-darwin-x64": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-darwin-x64/-/baml-darwin-x64-0.219.0.tgz", + "integrity": "sha512-t7pnbVT3KEE3+stOrW7nATdipCWK8OK1gTV8qiD4pN8lZwGMtJpzrVvgWeycroWCaeJ8tNsQA/V2FyFU9pqjkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-arm64-gnu": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-arm64-gnu/-/baml-linux-arm64-gnu-0.219.0.tgz", + "integrity": "sha512-a3ikRhlOdX+Lb08TaZU4k1Qb55mAHMuwArxpM1No/ltc0ylvo9EbJ4Nv0ZKemEHdngM6udagwdDxohpNzR1QTg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-arm64-musl": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-arm64-musl/-/baml-linux-arm64-musl-0.219.0.tgz", + "integrity": "sha512-KjwJL5aXf4XvwpX3RwMau19XcHDcffhbiTX+tk41kGC4LZl7UXB4FgnOBtSOMvWtHj2f4cIN81+yCZBRsMxdNg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-x64-gnu": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-x64-gnu/-/baml-linux-x64-gnu-0.219.0.tgz", + "integrity": "sha512-JWMhzx1LDfCPwDbOUQbFaPgNb7TUvLCb4+1TjRAJR+Z2Nl08RqNWUUfFctMEzsN6HrcHVOZ0xSl/hJRxN8OcTA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-linux-x64-musl": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-linux-x64-musl/-/baml-linux-x64-musl-0.219.0.tgz", + "integrity": "sha512-lPosn6eJqI+8EjMI71AUEb3B9uk4dUfQgerbDyu6FvNMWx+HZDPB4UWl66+OlEeMA2Gb6DUp3LLE/Jl9/KDKSg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-win32-arm64-msvc": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-win32-arm64-msvc/-/baml-win32-arm64-msvc-0.219.0.tgz", + "integrity": "sha512-djy9pn8s1Ogr3ovTDxDcOiM1YOhkBwNsBnlZDQzrIdR/LgSh/GGmrwW5Ks3gfa2NvvNOIFPccdUp0nDZZNsjfQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@boundaryml/baml-win32-x64-msvc": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@boundaryml/baml-win32-x64-msvc/-/baml-win32-x64-msvc-0.219.0.tgz", + "integrity": "sha512-BJkgq/qyn3BcBZt3iyfYWirZyZrKiqa1J0OTWd45P8OQ8dJhRnU6JGcpXPTXNmkryXQFY+O4HA+lQBKA5SZ7kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@scarf/scarf": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", + "integrity": "sha512-xxeapPiUXdZAE3che6f3xogoJPeZgig6omHEy1rIY5WVsB3H2BHNnZH+gHG6x91SCWyQCzWGsuL2Hh3ClO5/qQ==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true + }, + "node_modules/fern-api": { + "version": "5.176.0", + "resolved": "https://registry.npmjs.org/fern-api/-/fern-api-5.176.0.tgz", + "integrity": "sha512-WEwNFW0+ZpN8PHxAlWJfnnUUofOl7blgwWISigIKH/5Ack79XQjVVDR4Zn+R3pk7JC+q/0KlFnfu+z36d2QfVg==", + "dev": true, + "bin": { + "fern": "cli.cjs" + }, + "optionalDependencies": { + "@boundaryml/baml": "^0.219.0" + } + } + } +} diff --git a/fern/package.json b/fern/package.json new file mode 100644 index 000000000..b7e9da289 --- /dev/null +++ b/fern/package.json @@ -0,0 +1,11 @@ +{ + "name": "cloudai-docs", + "private": true, + "scripts": { + "check": "fern check && fern docs md check && fern docs broken-links --strict", + "dev": "fern docs dev --legacy" + }, + "devDependencies": { + "fern-api": "5.176.0" + } +} diff --git a/fern/reference.yml b/fern/reference.yml new file mode 100644 index 000000000..e5573d54b --- /dev/null +++ b/fern/reference.yml @@ -0,0 +1,214 @@ +references: +- symbol: cloudai.systems.slurm.slurm_system.SlurmSystem + exclude: + - cmd_shell + - group_allocated + - supports_gpu_directives_cache + fields_only: true +- symbol: cloudai.systems.slurm.slurm_system.SlurmPartition + exclude: + - slurm_nodes + fields_only: true +- symbol: cloudai.systems.slurm.slurm_system.SlurmGroup + exclude: [] + fields_only: true +- symbol: cloudai.systems.slurm.slurm_system.DataRepositoryConfig + exclude: [] + fields_only: true +- symbol: cloudai.systems.kubernetes.kubernetes_system.KubernetesSystem + exclude: [] + fields_only: true +- symbol: cloudai.systems.runai.runai_system.RunAISystem + exclude: + - nodes + fields_only: true +- symbol: cloudai.systems.runai.runai_node.RunAINode + exclude: [] + fields_only: true +- symbol: cloudai.systems.lsf.lsf_system.LSFSystem + exclude: + - cmd_shell + fields_only: true +- symbol: cloudai.systems.lsf.lsf_system.LSFQueue + exclude: + - lsf_nodes + fields_only: true +- symbol: cloudai.systems.lsf.lsf_system.LSFGroup + exclude: [] + fields_only: true +- symbol: cloudai.systems.standalone.standalone_system.StandaloneSystem + exclude: + - cmd_shell + fields_only: true +- symbol: cloudai.report_generator.training.models.TrainingResults + exclude: + - __init__ + fields_only: false +- symbol: cloudai.report_generator.training.models.TrainingConfig + exclude: + - __init__ + fields_only: false +- symbol: cloudai.report_generator.training.models.TrainingStep + exclude: + - __init__ + fields_only: false +- symbol: cloudai.report_generator.training.models.StepAggregation + exclude: + - __init__ + - from_steps + fields_only: false +- symbol: cloudai.report_generator.training.models.MetricStats + exclude: + - __init__ + - from_values + fields_only: false +- symbol: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.aiconfig.standalone_command_gen_strategy.AiconfiguratorStandaloneCommandGenStrategy + exclude: [] + fields_only: false +- symbol: cloudai.workloads.aiconfig.report_generation_strategy.AiconfiguratorReportGenerationStrategy + exclude: [] + fields_only: false +- symbol: cloudai.workloads.bash_cmd.bash_cmd.BashCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.bash_cmd.bash_cmd.BashCmdTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.ddlb.ddlb.DDLBCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.ddlb.ddlb.DDLBTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.deepep.deepep.DeepEPCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.deepep.deepep.DeepEPTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.fio.fio.FioCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.fio.fio.FioTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nccl_test.nccl.NCCLCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nccl_test.nccl.NCCLTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nemo_run.nemo_run.NeMoRunCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nemo_run.nemo_run.NeMoRunTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_ep.nixl_ep.NixlEPCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_ep.nixl_ep.NixlEPTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.osu_bench.osu_bench.OSUBenchCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.osu_bench.osu_bench.OSUBenchTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.sglang.sglang.SglangArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.sglang.sglang.SglangCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.sglang.sglang.SglangBenchCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.sglang.sglang.SglangSemanticEvalCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.sglang.sglang.SglangTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.sleep.sleep.SleepCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.sleep.sleep.SleepTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.slurm_container.slurm_container.SlurmContainerCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.slurm_container.slurm_container.SlurmContainerTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.ucc_test.ucc.UCCCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.ucc_test.ucc.UCCTestDefinition + exclude: [] + fields_only: false +- symbol: cloudai.workloads.vllm.vllm.VllmArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.vllm.vllm.VllmCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.vllm.vllm.VllmBenchCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.vllm.vllm.VllmSemanticEvalCmdArgs + exclude: [] + fields_only: false +- symbol: cloudai.workloads.vllm.vllm.VllmTestDefinition + exclude: [] + fields_only: false diff --git a/pyproject.toml b/pyproject.toml index 8088b8b5c..012b501b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,20 +68,8 @@ classifiers = [ rl = ["gymnasium~=1.2"] s3 = ["boto3~=1.40"] docs = [ - "sphinx~=8.1", - "nvidia-sphinx-theme~=0.0.8", - "sphinxext-opengraph~=0.13", - "sphinx-design~=0.6", - "sphinxcontrib-mermaid~=2.0", - "sphinx-copybutton~=0.5", - "autodoc-pydantic~=2.2", - ] - docs-cms = [ - "sphinx~=8.1", - "sphinx-rtd-theme~=3.1", - "sphinx-copybutton~=0.5", - "sphinxcontrib-mermaid~=2.0", - "autodoc-pydantic~=2.2", + "griffelib==2.4.1; python_version >= '3.11'", + "griffe-pydantic==1.3.2; python_version >= '3.11'", ] [build-system] diff --git a/src/cloudai/models/__init__.py b/src/cloudai/models/__init__.py new file mode 100644 index 000000000..41c6aaa6e --- /dev/null +++ b/src/cloudai/models/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cloudai/models/output.py b/src/cloudai/models/output.py index 44d49bef2..a3175560d 100644 --- a/src/cloudai/models/output.py +++ b/src/cloudai/models/output.py @@ -76,7 +76,7 @@ class Test(pydantic.BaseModel): class Experiment(pydantic.BaseModel): - """Full scenario snapshot; changes to this model tree may require updating doc/reporting.rst.""" + """Full scenario snapshot; changes to this model tree may require updating doc/reporting.mdx.""" id: str name: str diff --git a/src/cloudai/workloads/common/__init__.py b/src/cloudai/workloads/common/__init__.py new file mode 100644 index 000000000..41c6aaa6e --- /dev/null +++ b/src/cloudai/workloads/common/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/doc/test_generate_reference.py b/tests/doc/test_generate_reference.py new file mode 100644 index 000000000..b7d281f9e --- /dev/null +++ b/tests/doc/test_generate_reference.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression coverage for the Python-to-Fern reference build.""" + +import importlib.util +from pathlib import Path + +import pytest + +griffe = pytest.importorskip("griffe") +pytest.importorskip("griffe_pydantic") +pytest.importorskip("jinja2") + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("generate_reference", ROOT / "doc/generate_reference.py") +assert SPEC is not None and SPEC.loader is not None +reference = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(reference) + + +@pytest.fixture(scope="module") +def loader(): + result = griffe.GriffeLoader( + search_paths=[str(ROOT / "src")], + extensions=griffe.load_extensions("griffe_pydantic"), + docstring_parser=griffe.Parser.google, + ) + result.load("cloudai") + result.resolve_aliases(external=True) + return result + + +@pytest.mark.parametrize( + "symbol", + [ + "cloudai.systems.slurm.slurm_system.SlurmSystem", + "cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchCmdArgs", + "cloudai.workloads.vllm.vllm.VllmArgs", + ], +) +def test_all_model_fields_and_descriptions(loader, symbol): + result = reference.describe_class(symbol, loader.modules_collection[symbol]) + actual = {field["name"]: field for field in result["fields"]} + model = reference.load_class(symbol) + assert set(actual) == set(model.model_fields) + for name, field in model.model_fields.items(): + assert actual[name]["required"] == field.is_required() + if field.description: + assert actual[name]["description"] == field.description + + +def test_runtime_default_and_constraints(loader): + symbol = "cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchCmdArgs" + result = reference.describe_class(symbol, loader.modules_collection[symbol]) + fields = {field["name"]: field for field in result["fields"]} + assert fields["etcd_endpoints"]["default"] == "'http://$NIXL_ETCD_ENDPOINTS'" + assert fields["asio_port"]["default"] == "12345" + assert fields["asio_port"]["constraints"] == ["Ge(ge=1)", "Le(le=65535)"] + assert fields["docker_image_url"]["required"] + + +def test_system_exclusions_and_factory(loader): + symbol = "cloudai.systems.slurm.slurm_system.SlurmSystem" + result = reference.describe_class(symbol, loader.modules_collection[symbol], ["cmd_shell"], fields_only=True) + fields = {field["name"]: field for field in result["fields"]} + assert "cmd_shell" not in fields + assert fields["extra_sbatch_args"]["factory"] == "list" + assert not fields["extra_sbatch_args"]["required"] + assert not result["members"] + + +def test_dataclass_fields(loader): + symbol = "cloudai.report_generator.training.models.TrainingConfig" + result = reference.describe_class(symbol, loader.modules_collection[symbol], ["__init__"]) + fields = {field["name"]: field for field in result["fields"]} + assert len(fields) == 47 + assert fields["test_id"]["required"] + assert fields["container_image"]["default"] == "''" + assert all(member["name"] != "__init__" for member in result["members"]) + + +def test_mdx_prose_escaping(): + assert reference.mdx_text("x < y and {name}") == "x < y and {name}" diff --git a/uv.lock b/uv.lock index 5a5b2e668..2e338c01f 100644 --- a/uv.lock +++ b/uv.lock @@ -17,27 +17,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" }, ] -[[package]] -name = "accessible-pygments" -version = "0.0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bc/c1/bbac6a50d02774f91572938964c582fff4270eee73ab822a4aeea4d8b11b/accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872", size = 1377899, upload-time = "2024-05-10T11:23:10.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7", size = 1395903, upload-time = "2024-05-10T11:23:08.421Z" }, -] - -[[package]] -name = "alabaster" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, -] - [[package]] name = "annotated-doc" version = "0.0.4" @@ -71,43 +50,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] -[[package]] -name = "autodoc-pydantic" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/df/87120e2195f08d760bc5cf8a31cfa2381a6887517aa89453b23f1ae3354f/autodoc_pydantic-2.2.0-py3-none-any.whl", hash = "sha256:8c6a36fbf6ed2700ea9c6d21ea76ad541b621fbdf16b5a80ee04673548af4d95", size = 34001, upload-time = "2024-04-27T10:57:00.542Z" }, -] - -[[package]] -name = "babel" -version = "2.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, -] - -[[package]] -name = "beautifulsoup4" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, -] - [[package]] name = "bokeh" version = "3.8.2" @@ -336,23 +278,8 @@ dev = [ { name = "vulture" }, ] docs = [ - { name = "autodoc-pydantic" }, - { name = "nvidia-sphinx-theme" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "sphinx-copybutton" }, - { name = "sphinx-design", version = "0.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx-design", version = "0.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-mermaid" }, - { name = "sphinxext-opengraph" }, -] -docs-cms = [ - { name = "autodoc-pydantic" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "sphinx-copybutton" }, - { name = "sphinx-rtd-theme" }, - { name = "sphinxcontrib-mermaid" }, + { name = "griffe-pydantic", marker = "python_full_version >= '3.11'" }, + { name = "griffelib", marker = "python_full_version >= '3.11'" }, ] rl = [ { name = "gymnasium" }, @@ -363,8 +290,6 @@ s3 = [ [package.metadata] requires-dist = [ - { name = "autodoc-pydantic", marker = "extra == 'docs'", specifier = "~=2.2" }, - { name = "autodoc-pydantic", marker = "extra == 'docs-cms'", specifier = "~=2.2" }, { name = "bokeh", specifier = "~=3.8" }, { name = "boto3", marker = "extra == 'dev'", specifier = "~=1.40" }, { name = "boto3", marker = "extra == 's3'", specifier = "~=1.40" }, @@ -372,6 +297,8 @@ requires-dist = [ { name = "certifi", specifier = ">=2026.10.19" }, { name = "click", specifier = "~=8.3" }, { name = "defusedxml", marker = "extra == 'dev'", specifier = "~=0.7" }, + { name = "griffe-pydantic", marker = "python_full_version >= '3.11' and extra == 'docs'", specifier = "==1.3.2" }, + { name = "griffelib", marker = "python_full_version >= '3.11' and extra == 'docs'", specifier = "==2.4.1" }, { name = "gymnasium", marker = "extra == 'dev'", specifier = "~=1.2" }, { name = "gymnasium", marker = "extra == 'rl'", specifier = "~=1.2" }, { name = "huggingface-hub", specifier = "~=1.4" }, @@ -379,7 +306,6 @@ requires-dist = [ { name = "jinja2", specifier = "~=3.1.6" }, { name = "kubernetes", specifier = "~=35.0" }, { name = "numpy", marker = "python_full_version >= '3.14'", specifier = ">=2.4.6" }, - { name = "nvidia-sphinx-theme", marker = "extra == 'docs'", specifier = "~=0.0.8" }, { name = "pandas", specifier = "~=2.3" }, { name = "pandas-stubs", marker = "extra == 'dev'", specifier = "~=2.3" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "~=4.0" }, @@ -392,15 +318,6 @@ requires-dist = [ { name = "requests", specifier = "~=2.33" }, { name = "rich", specifier = "~=14.3" }, { name = "ruff", marker = "extra == 'dev'", specifier = "~=0.15" }, - { name = "sphinx", marker = "extra == 'docs'", specifier = "~=8.1" }, - { name = "sphinx", marker = "extra == 'docs-cms'", specifier = "~=8.1" }, - { name = "sphinx-copybutton", marker = "extra == 'docs'", specifier = "~=0.5" }, - { name = "sphinx-copybutton", marker = "extra == 'docs-cms'", specifier = "~=0.5" }, - { name = "sphinx-design", marker = "extra == 'docs'", specifier = "~=0.6" }, - { name = "sphinx-rtd-theme", marker = "extra == 'docs-cms'", specifier = "~=3.1" }, - { name = "sphinxcontrib-mermaid", marker = "extra == 'docs'", specifier = "~=2.0" }, - { name = "sphinxcontrib-mermaid", marker = "extra == 'docs-cms'", specifier = "~=2.0" }, - { name = "sphinxext-opengraph", marker = "extra == 'docs'", specifier = "~=0.13" }, { name = "taplo", marker = "extra == 'dev'", specifier = "~=0.9.3" }, { name = "tbparse", specifier = "~=0.0.9" }, { name = "tenacity", specifier = "~=9.1" }, @@ -409,7 +326,7 @@ requires-dist = [ { name = "vulture", marker = "extra == 'dev'", specifier = "==2.14" }, { name = "websockets", specifier = "~=16.0" }, ] -provides-extras = ["dev", "rl", "s3", "docs", "docs-cms"] +provides-extras = ["dev", "rl", "s3", "docs"] [[package]] name = "cloudpickle" @@ -723,15 +640,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/2b/238f02a4927fa2e27aa995371d240a468b52e42eae44a678434aff736b38/distlib-0.4.4-py2.py3-none-any.whl", hash = "sha256:cf09d55b1e85a89576b036aeb71550bfd68320f3f21761ca329eadadad918638", size = 472099, upload-time = "2026-10-16T13:56:32.053Z" }, ] -[[package]] -name = "docutils" -version = "0.21.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444, upload-time = "2024-04-23T18:57:18.24Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, -] - [[package]] name = "durationpy" version = "0.10" @@ -797,6 +705,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, ] +[[package]] +name = "griffe-pydantic" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffelib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/18/5420f9df75e45717075c82c0b7c4dd1a23ce6987051ffd48107df2266d13/griffe_pydantic-1.3.2.tar.gz", hash = "sha256:4d7cddda2d30ef7814cd3d78af854d6fcc14f3f170bd07360babbec62673d4ca", size = 40000, upload-time = "2026-10-06T09:08:06.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/57/157c6d57909082eb674fe080da72aff1ae7a87b7af2939ae9aceceb76783/griffe_pydantic-1.3.2-py3-none-any.whl", hash = "sha256:5bb5c61abb55556427dddcb43f187d734d96da34d3d35e218674e40ae1747a2a", size = 20765, upload-time = "2026-10-06T09:08:05.556Z" }, +] + +[[package]] +name = "griffelib" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/e2/0f5d8ed08766bc6dcd3eebf5a6623d83cbca33a6e4c90aef57156ce745ad/griffelib-2.4.1.tar.gz", hash = "sha256:40c028dd2e623f464423f0e11e3c553158f1cf4f1368239a370534d73e1407ff", size = 245153, upload-time = "2026-10-22T11:20:03.705Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/02/1156d5f5307f573b901b14373848b84e69ca6cab09169b8cc97d16a181f9/griffelib-2.4.1-py3-none-any.whl", hash = "sha256:76bc562f936ad1ce198d2cf6d0731bd5c92a7d934b69ef828ef22ee0ffdb62ce", size = 172983, upload-time = "2026-10-22T11:19:59.843Z" }, +] + [[package]] name = "grimp" version = "3.14" @@ -1099,15 +1028,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] -[[package]] -name = "imagesize" -version = "1.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a", size = 1280026, upload-time = "2022-07-01T12:21:05.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769, upload-time = "2022-07-01T12:21:02.467Z" }, -] - [[package]] name = "import-linter" version = "2.10" @@ -1598,19 +1518,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/6a/d3a169aaf8536cf228d56a09e04bcb713a2fe4410d4e2105b9419b5a9c89/numpy-2.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:016623417bb330d719d579daf2d6b9a01ddc52e41a9ed61a47f39fde46dcd865", size = 10686451, upload-time = "2026-06-21T20:57:49.313Z" }, ] -[[package]] -name = "nvidia-sphinx-theme" -version = "0.0.9.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydata-sphinx-theme" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/79/017fab2f7167a9a9795665f894d04f77aafceca80821b51589bb4b23ff5c/nvidia_sphinx_theme-0.0.9.post1-py3-none-any.whl", hash = "sha256:21ca60206dff2f380d7783d64bbaf71a5b9cacae53c7d0686f089c16b5a3d45a", size = 143816, upload-time = "2025-11-09T23:16:55.719Z" }, -] - [[package]] name = "oauthlib" version = "4.0.0" @@ -1985,40 +1892,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] -[[package]] -name = "pydantic-settings" -version = "2.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, -] - -[[package]] -name = "pydata-sphinx-theme" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "accessible-pygments" }, - { name = "babel" }, - { name = "beautifulsoup4" }, - { name = "docutils" }, - { name = "pygments" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/00/20/bb50f9de3a6de69e6abd6b087b52fa2418a0418b19597601605f855ad044/pydata_sphinx_theme-0.16.1.tar.gz", hash = "sha256:a08b7f0b7f70387219dc659bff0893a7554d5eb39b59d3b8ef37b8401b7642d7", size = 2412693, upload-time = "2024-12-17T10:53:39.537Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/0d/8ba33fa83a7dcde13eb3c1c2a0c1cc29950a048bfed6d9b0d8b6bd710b4c/pydata_sphinx_theme-0.16.1-py3-none-any.whl", hash = "sha256:225331e8ac4b32682c18fcac5a57a6f717c4e632cea5dd0e247b55155faeccde", size = 6723264, upload-time = "2024-12-17T10:53:35.645Z" }, -] - [[package]] name = "pygments" version = "2.20.0" @@ -2124,15 +1997,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/91/1aa70d3dfcab0dbcf88a2cbc3bce7f7c69562287a90216e617eaa4937e25/python_discovery-1.6.4-py3-none-any.whl", hash = "sha256:530179cc51a1816d512b15635b3dc931295ae12efb183783ad31190a5d425d2b", size = 39385, upload-time = "2026-10-19T01:45:13.668Z" }, ] -[[package]] -name = "python-dotenv" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, -] - [[package]] name = "pytz" version = "2026.5" @@ -2247,27 +2111,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl", hash = "sha256:08e67c3e90884651da3239ea668222d19bea7b589149d8014a21c633420dbb69", size = 309963, upload-time = "2026-02-01T16:20:46.078Z" }, ] -[[package]] -name = "roman-numerals" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, -] - -[[package]] -name = "roman-numerals-py" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "roman-numerals" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/2c/daca29684cbe9fd4bc711f8246da3c10adca1ccc4d24436b17572eb2590e/roman_numerals_py-4.1.0-py3-none-any.whl", hash = "sha256:553114c1167141c1283a51743759723ecd05604a1b6b507225e91dc1a6df0780", size = 4547, upload-time = "2025-12-17T18:25:40.136Z" }, -] - [[package]] name = "ruff" version = "0.15.0" @@ -2332,243 +2175,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "snowballstemmer" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260", size = 123314, upload-time = "2026-06-03T00:56:40.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, -] - -[[package]] -name = "soupsieve" -version = "2.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/f1/93422647dd7e461f23d254e6b2bfa687a85b53aeb4903fcdbb74474d4584/soupsieve-2.9.tar.gz", hash = "sha256:acee8417325c5653e1377dc31eccad59eb82cbc65942afe6174c53b3aaad63fc", size = 122122, upload-time = "2026-07-19T01:35:18.425Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/d6/3185ab5ad1280319b31986898f3206dd7227cd75e293d4dba2a5e6bf27a0/soupsieve-2.9-py3-none-any.whl", hash = "sha256:a2b2c76d67df2382d245409fd71e321a571717e58463efa32ace87dcadac2c12", size = 37387, upload-time = "2026-07-19T01:35:17.106Z" }, -] - -[[package]] -name = "sphinx" -version = "8.1.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, - { name = "tomli" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/60/1ddff83a56d33aaf6f10ec8ce84b4c007d9368b21008876fceda7e7381ef/sphinx-8.1.3-py3-none-any.whl", hash = "sha256:09719015511837b76bf6e03e42eb7595ac8c2e41eeb9c29c5b755c6b677992a2", size = 3487125, upload-time = "2024-10-13T20:27:10.448Z" }, -] - -[[package]] -name = "sphinx" -version = "8.2.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version >= '3.12' and python_full_version < '3.14'", - "python_full_version == '3.11.*'", -] -dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "roman-numerals-py" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348", size = 8321876, upload-time = "2025-03-02T22:31:59.658Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/53/136e9eca6e0b9dc0e1962e2c908fbea2e5ac000c2a2fbd9a35797958c48b/sphinx-8.2.3-py3-none-any.whl", hash = "sha256:4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3", size = 3589741, upload-time = "2025-03-02T22:31:56.836Z" }, -] - -[[package]] -name = "sphinx-copybutton" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/2b/a964715e7f5295f77509e59309959f4125122d648f86b4fe7d70ca1d882c/sphinx-copybutton-0.5.2.tar.gz", hash = "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd", size = 23039, upload-time = "2023-04-14T08:10:22.998Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, -] - -[[package]] -name = "sphinx-design" -version = "0.6.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/69/b34e0cb5336f09c6866d53b4a19d76c227cdec1bbc7ac4de63ca7d58c9c7/sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632", size = 2193689, upload-time = "2024-08-02T13:48:44.277Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/43/65c0acbd8cc6f50195a3a1fc195c404988b15c67090e73c7a41a9f57d6bd/sphinx_design-0.6.1-py3-none-any.whl", hash = "sha256:b11f37db1a802a183d61b159d9a202314d4d2fe29c163437001324fe2f19549c", size = 2215338, upload-time = "2024-08-02T13:48:42.106Z" }, -] - -[[package]] -name = "sphinx-design" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version >= '3.12' and python_full_version < '3.14'", - "python_full_version == '3.11.*'", -] -dependencies = [ - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" } }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/7b/804f311da4663a4aecc6cf7abd83443f3d4ded970826d0c958edc77d4527/sphinx_design-0.7.0.tar.gz", hash = "sha256:d2a3f5b19c24b916adb52f97c5f00efab4009ca337812001109084a740ec9b7a", size = 2203582, upload-time = "2026-01-19T13:12:53.297Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/cf/45dd359f6ca0c3762ce0490f681da242f0530c49c81050c035c016bfdd3a/sphinx_design-0.7.0-py3-none-any.whl", hash = "sha256:f82bf179951d58f55dca78ab3706aeafa496b741a91b1911d371441127d64282", size = 2220350, upload-time = "2026-01-19T13:12:51.077Z" }, -] - -[[package]] -name = "sphinx-rtd-theme" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docutils" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-jquery" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/84/68/a1bfbf38c0f7bccc9b10bbf76b94606f64acb1552ae394f0b8285bfaea25/sphinx_rtd_theme-3.1.0.tar.gz", hash = "sha256:b44276f2c276e909239a4f6c955aa667aaafeb78597923b1c60babc76db78e4c", size = 7620915, upload-time = "2026-01-12T16:03:31.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/c7/b5c8015d823bfda1a346adb2c634a2101d50bb75d421eb6dcb31acd25ebc/sphinx_rtd_theme-3.1.0-py2.py3-none-any.whl", hash = "sha256:1785824ae8e6632060490f67cf3a72d404a85d2d9fc26bce3619944de5682b89", size = 7655617, upload-time = "2026-01-12T16:03:28.101Z" }, -] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, -] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, -] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, -] - -[[package]] -name = "sphinxcontrib-jquery" -version = "4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/de/f3/aa67467e051df70a6330fe7770894b3e4f09436dea6881ae0b4f3d87cad8/sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a", size = 122331, upload-time = "2023-03-14T15:01:01.944Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/85/749bd22d1a68db7291c89e2ebca53f4306c3f205853cf31e9de279034c3c/sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae", size = 121104, upload-time = "2023-03-14T15:01:00.356Z" }, -] - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, -] - -[[package]] -name = "sphinxcontrib-mermaid" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "pyyaml" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/96/a5/65a5c439cc14ba80483b9891e9350f11efb80cd3bdccb222f0c738068c78/sphinxcontrib_mermaid-2.0.0.tar.gz", hash = "sha256:cf4f7d453d001132eaba5d1fdf53d42049f02e913213cf8337427483bfca26f4", size = 18194, upload-time = "2026-01-13T17:13:42.563Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/de/bd96c69b62e967bffd02c6d89dfca9471b04e761c466725fc39746abf41d/sphinxcontrib_mermaid-2.0.0-py3-none-any.whl", hash = "sha256:59a73249bbee2c74b1a4db036f8e8899ade65982bdda6712cf22b4f4e9874bb5", size = 14055, upload-time = "2026-01-13T17:13:41.481Z" }, -] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, -] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, -] - -[[package]] -name = "sphinxext-opengraph" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/c0/eb6838e3bae624ce6c8b90b245d17e84252863150e95efdb88f92c8aa3fb/sphinxext_opengraph-0.13.0.tar.gz", hash = "sha256:103335d08567ad8468faf1425f575e3b698e9621f9323949a6c8b96d9793e80b", size = 1026875, upload-time = "2025-08-29T12:20:31.066Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/a4/66c1fd4f8fab88faf71cee04a945f9806ba0fef753f2cfc8be6353f64508/sphinxext_opengraph-0.13.0-py3-none-any.whl", hash = "sha256:936c07828edc9ad9a7b07908b29596dc84ed0b3ceaa77acdf51282d232d4d80e", size = 1004152, upload-time = "2025-08-29T12:20:29.072Z" }, -] - [[package]] name = "starlette" version = "1.3.1" From 0328ffad383bc670770d911363d2e0622338ff64 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Thu, 29 Oct 2026 14:49:49 +0100 Subject: [PATCH 2/6] Clean up Fern documentation contribution files Signed-off-by: Ivan Podkidyshev --- CONTRIBUTING.md | 18 +- doc/README.md | 20 ++ doc/generate_reference.py | 2 - fern/docs.yml | 234 +++++++-------- fern/reference.yml | 426 +++++++++++++-------------- tests/doc/test_generate_reference.py | 2 - 6 files changed, 351 insertions(+), 351 deletions(-) create mode 100644 doc/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 510993ba7..89fe6dc77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,23 +3,7 @@ Welcome to the CloudAI project! This document outlines the procedures and standa ## Documentation -Documentation is authored in `doc/*.mdx` and `doc/workloads/*.mdx` and built with Fern. -Keep the README separate. Navigation lives in `fern/docs.yml`. - -With Python 3.14, Node.js 24 and uv installed, run `make -C doc check` to generate the Python reference -and validate the documentation, or `make -C doc preview` to start a local preview. -After changing Python docstrings or model fields, rerun `make -C doc reference`. -The preview uses Fern's legacy renderer because the default Astro preview currently returns -404s for compiled MDX modules, causing page content to disappear after loading. - -Generated reference snippets live in `fern/generated/` and are not checked in. Add classes -and member exclusions to `fern/reference.yml`; include their snippets in authored pages with -``. The build reads Python objects -and renders `doc/templates/reference.mdx.jinja`; it does not transform authored MDX. - -CI validates the docs and produces a Fern source artifact. Publishing requires the provisioned -Fern instance URL (`FERN_DOCS_URL`) and the enterprise deployment environment. The default URL -is for local validation only; this repository does not publish automatically. +See [Documentation](doc/README.md) for authoring, validation, and local preview instructions. ## Contribution Process ### Identifying the Feature or Fix diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 000000000..f96b44010 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,20 @@ +# Documentation + +Documentation is authored in `doc/*.mdx` and `doc/workloads/*.mdx` and built with Fern. +Keep the root README separate. Navigation lives in `fern/docs.yml`. +Paths and commands below are relative to the repository root. + +With Python 3.14, Node.js 24 and uv installed, run `make -C doc check` to generate the Python reference +and validate the documentation, or `make -C doc preview` to start a local preview. +After changing Python docstrings or model fields, rerun `make -C doc reference`. +The preview uses Fern's legacy renderer because the default Astro preview currently returns +404s for compiled MDX modules, causing page content to disappear after loading. + +Generated reference snippets live in `fern/generated/` and are not checked in. Add classes +and member exclusions to `fern/reference.yml`; include their snippets in authored pages with +``. The build reads Python objects +and renders `doc/templates/reference.mdx.jinja`; it does not transform authored MDX. + +CI validates the docs and produces a Fern source artifact. Publishing requires the provisioned +Fern instance URL (`FERN_DOCS_URL`) and the enterprise deployment environment. The default URL +is for local validation only; this repository does not publish automatically. diff --git a/doc/generate_reference.py b/doc/generate_reference.py index de80f19dd..0ec19ea99 100644 --- a/doc/generate_reference.py +++ b/doc/generate_reference.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Generate Fern snippets from Python objects; authored MDX is never rewritten.""" - import dataclasses import html import importlib diff --git a/fern/docs.yml b/fern/docs.yml index 40671f633..5e2bdd3a7 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -1,125 +1,125 @@ # yaml-language-server: $schema=https://schema.buildwithfern.dev/docs-yml.json instances: -- url: ${FERN_DOCS_URL} + - url: ${FERN_DOCS_URL} title: CloudAI settings: substitute-env-vars: true global-theme: nvidia navbar-links: -- type: github - value: https://github.com/NVIDIA/cloudai + - type: github + value: https://github.com/NVIDIA/cloudai navigation: -- page: CloudAI Benchmark Framework - path: ../doc/index.mdx - slug: / -- page: Tutorial - path: ../doc/Tutorial.mdx - slug: Tutorial -- page: User Guide - path: ../doc/USER_GUIDE.mdx - slug: USER_GUIDE -- page: Python API - path: ../doc/api.mdx - slug: api -- section: Reporting - skip-slug: true - contents: - - page: Reporting - path: ../doc/reporting.mdx - slug: reporting - - page: Reports - path: ../doc/reports.mdx - slug: reports - - page: Pareto Frontiers - path: ../doc/pareto-frontiers.mdx - slug: pareto-frontiers - - page: Training Report Schema - path: ../doc/training-report-schema.mdx - slug: training-report-schema -- page: Systems - path: ../doc/systems.mdx - slug: systems -- section: Workloads - skip-slug: true - contents: - - page: Workloads - path: ../doc/workloads/index.mdx - slug: workloads/index - - page: AI Dynamo - path: ../doc/workloads/ai_dynamo.mdx - slug: workloads/ai_dynamo - - page: AIConfigurator - path: ../doc/workloads/aiconfigurator.mdx - slug: workloads/aiconfigurator - - page: Bash Command - path: ../doc/workloads/bash_cmd.mdx - slug: workloads/bash_cmd - - page: Chakra Replay - path: ../doc/workloads/chakra_replay.mdx - slug: workloads/chakra_replay - - page: DDLB - path: ../doc/workloads/ddlb.mdx - slug: workloads/ddlb - - page: DeepEP Benchmark - path: ../doc/workloads/deepep.mdx - slug: workloads/deepep - - page: DynamoMocker - path: ../doc/workloads/dynamo_mocker.mdx - slug: workloads/dynamo_mocker - - page: fio - path: ../doc/workloads/fio.mdx - slug: workloads/fio - - page: JaxToolbox workloads (DEPRECATED) - path: ../doc/workloads/jax_toolbox.mdx - slug: workloads/jax_toolbox - - page: MegatronBridge - path: ../doc/workloads/megatron_bridge.mdx - slug: workloads/megatron_bridge - - page: NCCL - path: ../doc/workloads/nccl.mdx - slug: workloads/nccl - - page: NeMo v1.0 aka NemoLauncher (DEPRECATED) - path: ../doc/workloads/nemo_launcher.mdx - slug: workloads/nemo_launcher - - page: Nemo Run - path: ../doc/workloads/nemo_run.mdx - slug: workloads/nemo_run - - page: NIXL Bench - path: ../doc/workloads/nixl_bench.mdx - slug: workloads/nixl_bench - - page: NIXL EP - path: ../doc/workloads/nixl_ep.mdx - slug: workloads/nixl_ep - - page: NIXL KVBench - path: ../doc/workloads/nixl_kvbench.mdx - slug: workloads/nixl_kvbench - - page: NIXL CTPerf - path: ../doc/workloads/nixl_perftest.mdx - slug: workloads/nixl_perftest - - page: OSU - path: ../doc/workloads/osu.mdx - slug: workloads/osu - - page: SGLang - path: ../doc/workloads/sglang.mdx - slug: workloads/sglang - - page: Sleep - path: ../doc/workloads/sleep.mdx - slug: workloads/sleep - - page: Slurm Container - path: ../doc/workloads/slurm_container.mdx - slug: workloads/slurm_container - - page: UCC - path: ../doc/workloads/ucc.mdx - slug: workloads/ucc - - page: vLLM - path: ../doc/workloads/vllm.mdx - slug: workloads/vllm - - page: Installation Requirements - path: ../doc/workloads/workloads_requirements_installation.mdx - slug: workloads/workloads_requirements_installation -- page: Development - path: ../doc/DEV.mdx - slug: DEV -- page: Troubleshooting - path: ../doc/troubleshooting.mdx - slug: troubleshooting + - page: CloudAI Benchmark Framework + path: ../doc/index.mdx + slug: / + - page: Tutorial + path: ../doc/Tutorial.mdx + slug: Tutorial + - page: User Guide + path: ../doc/USER_GUIDE.mdx + slug: USER_GUIDE + - page: Python API + path: ../doc/api.mdx + slug: api + - section: Reporting + skip-slug: true + contents: + - page: Reporting + path: ../doc/reporting.mdx + slug: reporting + - page: Reports + path: ../doc/reports.mdx + slug: reports + - page: Pareto Frontiers + path: ../doc/pareto-frontiers.mdx + slug: pareto-frontiers + - page: Training Report Schema + path: ../doc/training-report-schema.mdx + slug: training-report-schema + - page: Systems + path: ../doc/systems.mdx + slug: systems + - section: Workloads + skip-slug: true + contents: + - page: Workloads + path: ../doc/workloads/index.mdx + slug: workloads/index + - page: AI Dynamo + path: ../doc/workloads/ai_dynamo.mdx + slug: workloads/ai_dynamo + - page: AIConfigurator + path: ../doc/workloads/aiconfigurator.mdx + slug: workloads/aiconfigurator + - page: Bash Command + path: ../doc/workloads/bash_cmd.mdx + slug: workloads/bash_cmd + - page: Chakra Replay + path: ../doc/workloads/chakra_replay.mdx + slug: workloads/chakra_replay + - page: DDLB + path: ../doc/workloads/ddlb.mdx + slug: workloads/ddlb + - page: DeepEP Benchmark + path: ../doc/workloads/deepep.mdx + slug: workloads/deepep + - page: DynamoMocker + path: ../doc/workloads/dynamo_mocker.mdx + slug: workloads/dynamo_mocker + - page: fio + path: ../doc/workloads/fio.mdx + slug: workloads/fio + - page: JaxToolbox workloads (DEPRECATED) + path: ../doc/workloads/jax_toolbox.mdx + slug: workloads/jax_toolbox + - page: MegatronBridge + path: ../doc/workloads/megatron_bridge.mdx + slug: workloads/megatron_bridge + - page: NCCL + path: ../doc/workloads/nccl.mdx + slug: workloads/nccl + - page: NeMo v1.0 aka NemoLauncher (DEPRECATED) + path: ../doc/workloads/nemo_launcher.mdx + slug: workloads/nemo_launcher + - page: Nemo Run + path: ../doc/workloads/nemo_run.mdx + slug: workloads/nemo_run + - page: NIXL Bench + path: ../doc/workloads/nixl_bench.mdx + slug: workloads/nixl_bench + - page: NIXL EP + path: ../doc/workloads/nixl_ep.mdx + slug: workloads/nixl_ep + - page: NIXL KVBench + path: ../doc/workloads/nixl_kvbench.mdx + slug: workloads/nixl_kvbench + - page: NIXL CTPerf + path: ../doc/workloads/nixl_perftest.mdx + slug: workloads/nixl_perftest + - page: OSU + path: ../doc/workloads/osu.mdx + slug: workloads/osu + - page: SGLang + path: ../doc/workloads/sglang.mdx + slug: workloads/sglang + - page: Sleep + path: ../doc/workloads/sleep.mdx + slug: workloads/sleep + - page: Slurm Container + path: ../doc/workloads/slurm_container.mdx + slug: workloads/slurm_container + - page: UCC + path: ../doc/workloads/ucc.mdx + slug: workloads/ucc + - page: vLLM + path: ../doc/workloads/vllm.mdx + slug: workloads/vllm + - page: Installation Requirements + path: ../doc/workloads/workloads_requirements_installation.mdx + slug: workloads/workloads_requirements_installation + - page: Development + path: ../doc/DEV.mdx + slug: DEV + - page: Troubleshooting + path: ../doc/troubleshooting.mdx + slug: troubleshooting diff --git a/fern/reference.yml b/fern/reference.yml index e5573d54b..735cc7b1e 100644 --- a/fern/reference.yml +++ b/fern/reference.yml @@ -1,214 +1,214 @@ references: -- symbol: cloudai.systems.slurm.slurm_system.SlurmSystem - exclude: - - cmd_shell - - group_allocated - - supports_gpu_directives_cache - fields_only: true -- symbol: cloudai.systems.slurm.slurm_system.SlurmPartition - exclude: - - slurm_nodes - fields_only: true -- symbol: cloudai.systems.slurm.slurm_system.SlurmGroup - exclude: [] - fields_only: true -- symbol: cloudai.systems.slurm.slurm_system.DataRepositoryConfig - exclude: [] - fields_only: true -- symbol: cloudai.systems.kubernetes.kubernetes_system.KubernetesSystem - exclude: [] - fields_only: true -- symbol: cloudai.systems.runai.runai_system.RunAISystem - exclude: - - nodes - fields_only: true -- symbol: cloudai.systems.runai.runai_node.RunAINode - exclude: [] - fields_only: true -- symbol: cloudai.systems.lsf.lsf_system.LSFSystem - exclude: - - cmd_shell - fields_only: true -- symbol: cloudai.systems.lsf.lsf_system.LSFQueue - exclude: - - lsf_nodes - fields_only: true -- symbol: cloudai.systems.lsf.lsf_system.LSFGroup - exclude: [] - fields_only: true -- symbol: cloudai.systems.standalone.standalone_system.StandaloneSystem - exclude: - - cmd_shell - fields_only: true -- symbol: cloudai.report_generator.training.models.TrainingResults - exclude: - - __init__ - fields_only: false -- symbol: cloudai.report_generator.training.models.TrainingConfig - exclude: - - __init__ - fields_only: false -- symbol: cloudai.report_generator.training.models.TrainingStep - exclude: - - __init__ - fields_only: false -- symbol: cloudai.report_generator.training.models.StepAggregation - exclude: - - __init__ - - from_steps - fields_only: false -- symbol: cloudai.report_generator.training.models.MetricStats - exclude: - - __init__ - - from_values - fields_only: false -- symbol: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.aiconfig.standalone_command_gen_strategy.AiconfiguratorStandaloneCommandGenStrategy - exclude: [] - fields_only: false -- symbol: cloudai.workloads.aiconfig.report_generation_strategy.AiconfiguratorReportGenerationStrategy - exclude: [] - fields_only: false -- symbol: cloudai.workloads.bash_cmd.bash_cmd.BashCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.bash_cmd.bash_cmd.BashCmdTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.ddlb.ddlb.DDLBCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.ddlb.ddlb.DDLBTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.deepep.deepep.DeepEPCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.deepep.deepep.DeepEPTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.fio.fio.FioCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.fio.fio.FioTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nccl_test.nccl.NCCLCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nccl_test.nccl.NCCLTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nemo_run.nemo_run.NeMoRunCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nemo_run.nemo_run.NeMoRunTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_ep.nixl_ep.NixlEPCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_ep.nixl_ep.NixlEPTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.osu_bench.osu_bench.OSUBenchCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.osu_bench.osu_bench.OSUBenchTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.sglang.sglang.SglangArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.sglang.sglang.SglangCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.sglang.sglang.SglangBenchCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.sglang.sglang.SglangSemanticEvalCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.sglang.sglang.SglangTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.sleep.sleep.SleepCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.sleep.sleep.SleepTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.slurm_container.slurm_container.SlurmContainerCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.slurm_container.slurm_container.SlurmContainerTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.ucc_test.ucc.UCCCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.ucc_test.ucc.UCCTestDefinition - exclude: [] - fields_only: false -- symbol: cloudai.workloads.vllm.vllm.VllmArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.vllm.vllm.VllmCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.vllm.vllm.VllmBenchCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.vllm.vllm.VllmSemanticEvalCmdArgs - exclude: [] - fields_only: false -- symbol: cloudai.workloads.vllm.vllm.VllmTestDefinition - exclude: [] - fields_only: false + - symbol: cloudai.systems.slurm.slurm_system.SlurmSystem + exclude: + - cmd_shell + - group_allocated + - supports_gpu_directives_cache + fields_only: true + - symbol: cloudai.systems.slurm.slurm_system.SlurmPartition + exclude: + - slurm_nodes + fields_only: true + - symbol: cloudai.systems.slurm.slurm_system.SlurmGroup + exclude: [] + fields_only: true + - symbol: cloudai.systems.slurm.slurm_system.DataRepositoryConfig + exclude: [] + fields_only: true + - symbol: cloudai.systems.kubernetes.kubernetes_system.KubernetesSystem + exclude: [] + fields_only: true + - symbol: cloudai.systems.runai.runai_system.RunAISystem + exclude: + - nodes + fields_only: true + - symbol: cloudai.systems.runai.runai_node.RunAINode + exclude: [] + fields_only: true + - symbol: cloudai.systems.lsf.lsf_system.LSFSystem + exclude: + - cmd_shell + fields_only: true + - symbol: cloudai.systems.lsf.lsf_system.LSFQueue + exclude: + - lsf_nodes + fields_only: true + - symbol: cloudai.systems.lsf.lsf_system.LSFGroup + exclude: [] + fields_only: true + - symbol: cloudai.systems.standalone.standalone_system.StandaloneSystem + exclude: + - cmd_shell + fields_only: true + - symbol: cloudai.report_generator.training.models.TrainingResults + exclude: + - __init__ + fields_only: false + - symbol: cloudai.report_generator.training.models.TrainingConfig + exclude: + - __init__ + fields_only: false + - symbol: cloudai.report_generator.training.models.TrainingStep + exclude: + - __init__ + fields_only: false + - symbol: cloudai.report_generator.training.models.StepAggregation + exclude: + - __init__ + - from_steps + fields_only: false + - symbol: cloudai.report_generator.training.models.MetricStats + exclude: + - __init__ + - from_values + fields_only: false + - symbol: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.ai_dynamo.ai_dynamo.AIDynamoTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.aiconfig.aiconfigurator.AiconfiguratorTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.aiconfig.standalone_command_gen_strategy.AiconfiguratorStandaloneCommandGenStrategy + exclude: [] + fields_only: false + - symbol: cloudai.workloads.aiconfig.report_generation_strategy.AiconfiguratorReportGenerationStrategy + exclude: [] + fields_only: false + - symbol: cloudai.workloads.bash_cmd.bash_cmd.BashCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.bash_cmd.bash_cmd.BashCmdTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.chakra_replay.chakra_replay.ChakraReplayTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.ddlb.ddlb.DDLBCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.ddlb.ddlb.DDLBTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.deepep.deepep.DeepEPCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.deepep.deepep.DeepEPTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.dynamo_mocker.dynamo_mocker.DynamoMockerTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.fio.fio.FioCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.fio.fio.FioTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.megatron_bridge.megatron_bridge.MegatronBridgeTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nccl_test.nccl.NCCLCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nccl_test.nccl.NCCLTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nemo_run.nemo_run.NeMoRunCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nemo_run.nemo_run.NeMoRunTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_bench.nixl_bench.NIXLBenchTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_ep.nixl_ep.NixlEPCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_ep.nixl_ep.NixlEPTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_kvbench.nixl_kvbench.NIXLKVBenchTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.nixl_perftest.nixl_perftest.NixlPerftestTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.osu_bench.osu_bench.OSUBenchCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.osu_bench.osu_bench.OSUBenchTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.sglang.sglang.SglangArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.sglang.sglang.SglangCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.sglang.sglang.SglangBenchCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.sglang.sglang.SglangSemanticEvalCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.sglang.sglang.SglangTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.sleep.sleep.SleepCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.sleep.sleep.SleepTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.slurm_container.slurm_container.SlurmContainerCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.slurm_container.slurm_container.SlurmContainerTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.ucc_test.ucc.UCCCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.ucc_test.ucc.UCCTestDefinition + exclude: [] + fields_only: false + - symbol: cloudai.workloads.vllm.vllm.VllmArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.vllm.vllm.VllmCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.vllm.vllm.VllmBenchCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.vllm.vllm.VllmSemanticEvalCmdArgs + exclude: [] + fields_only: false + - symbol: cloudai.workloads.vllm.vllm.VllmTestDefinition + exclude: [] + fields_only: false diff --git a/tests/doc/test_generate_reference.py b/tests/doc/test_generate_reference.py index b7d281f9e..fa301bee1 100644 --- a/tests/doc/test_generate_reference.py +++ b/tests/doc/test_generate_reference.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Regression coverage for the Python-to-Fern reference build.""" - import importlib.util from pathlib import Path From ff61ffb312111338a1b938847ca8192808c6dbf8 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Thu, 29 Oct 2026 15:07:59 +0100 Subject: [PATCH 3/6] Fix copyright checks with Git 2.56 history traversal Signed-off-by: Ivan Podkidyshev --- doc/generate_reference.py | 2 +- src/cloudai/models/__init__.py | 2 +- src/cloudai/workloads/common/__init__.py | 2 +- tests/doc/test_generate_reference.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/generate_reference.py b/doc/generate_reference.py index 0ec19ea99..370624bd6 100644 --- a/doc/generate_reference.py +++ b/doc/generate_reference.py @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/cloudai/models/__init__.py b/src/cloudai/models/__init__.py index 41c6aaa6e..3ac11851d 100644 --- a/src/cloudai/models/__init__.py +++ b/src/cloudai/models/__init__.py @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/cloudai/workloads/common/__init__.py b/src/cloudai/workloads/common/__init__.py index 41c6aaa6e..3ac11851d 100644 --- a/src/cloudai/workloads/common/__init__.py +++ b/src/cloudai/workloads/common/__init__.py @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/doc/test_generate_reference.py b/tests/doc/test_generate_reference.py index fa301bee1..0bde03ea7 100644 --- a/tests/doc/test_generate_reference.py +++ b/tests/doc/test_generate_reference.py @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); From 709d24c25dd115cdbfa574d6ef15dbca8c452327 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 30 Oct 2026 14:12:44 +0100 Subject: [PATCH 4/6] Use published Fern routes for documentation links Signed-off-by: Ivan Podkidyshev --- doc/Tutorial.mdx | 16 ++++++------ doc/USER_GUIDE.mdx | 14 +++++------ doc/index.mdx | 36 +++++++++++++-------------- doc/pareto-frontiers.mdx | 2 +- doc/reporting.mdx | 18 +++++++------- doc/reports.mdx | 2 +- doc/systems.mdx | 10 ++++---- doc/workloads/index.mdx | 44 ++++++++++++++++----------------- doc/workloads/nemo_launcher.mdx | 2 +- 9 files changed, 72 insertions(+), 72 deletions(-) diff --git a/doc/Tutorial.mdx b/doc/Tutorial.mdx index 64c225d63..1f2d4dad6 100644 --- a/doc/Tutorial.mdx +++ b/doc/Tutorial.mdx @@ -2,14 +2,14 @@ This chapter outlines a tutorial on how to utilize the CloudAI framework. Please follow the steps in the same sequence to ensure successful execution: -- [Step 1: Creating a Docker Image](Tutorial.mdx#step-1-creating-docker-image) -- [Step 2: Preparing Configuration Files](Tutorial.mdx#step-2-preparing-configuration-files) -- [Step 3: Testing Definition](Tutorial.mdx#step-3-testing-definition) -- [Step 4: System Configuration](Tutorial.mdx#step-4-system-configuration) -- [Step 5: Testing Configuration](Tutorial.mdx#step-5-testing-configuration) -- [Step 6: Running Experiments](Tutorial.mdx#step-6-running-experiments) -- [Step 7: Generating Reports](Tutorial.mdx#step-7-generating-reports) -- [Test in Scenario](Tutorial.mdx#test-in-scenario) +- [Step 1: Creating a Docker Image](#step-1-creating-docker-image) +- [Step 2: Preparing Configuration Files](#step-2-preparing-configuration-files) +- [Step 3: Testing Definition](#step-3-testing-definition) +- [Step 4: System Configuration](#step-4-system-configuration) +- [Step 5: Testing Configuration](#step-5-testing-configuration) +- [Step 6: Running Experiments](#step-6-running-experiments) +- [Step 7: Generating Reports](#step-7-generating-reports) +- [Test in Scenario](#test-in-scenario) diff --git a/doc/USER_GUIDE.mdx b/doc/USER_GUIDE.mdx index e1fd00836..1fbc7c9f8 100644 --- a/doc/USER_GUIDE.mdx +++ b/doc/USER_GUIDE.mdx @@ -253,9 +253,9 @@ The HTTP Data Repository is currently supported for Slurm systems only. To enabl The following steps are required to configure the HTTP Data Repository: -- [Step 1: Update the System Schema File](USER_GUIDE.mdx#step-1-update-system-schema-file) -- [Step 2: Create the Credential File](USER_GUIDE.mdx#step-2-create-credential-file) -- [Step 3: Usage](USER_GUIDE.mdx#step-3-usage) +- [Step 1: Update the System Schema File](#step-1-update-system-schema-file) +- [Step 2: Create the Credential File](#step-2-create-credential-file) +- [Step 3: Usage](#step-3-usage) @@ -332,10 +332,10 @@ To run DeepSeek R1 tests in CloudAI, you must download the model weights in adva The following steps are required to download the DeepSeek weights: -- [Step 1: Installing NGC CLI](USER_GUIDE.mdx#step-1-installing-ngc-cli) -- [Step 2: Configuring NGC CLI](USER_GUIDE.mdx#step-2-configuring-ngc-cli) -- [Step 3: Downloading the Weights](USER_GUIDE.mdx#step-3-downloading-the-weights) -- [Step 4: Verifying the Download](USER_GUIDE.mdx#step-4-verifying-the-download) +- [Step 1: Installing NGC CLI](#step-1-installing-ngc-cli) +- [Step 2: Configuring NGC CLI](#step-2-configuring-ngc-cli) +- [Step 3: Downloading the Weights](#step-3-downloading-the-weights) +- [Step 4: Verifying the Download](#step-4-verifying-the-download) diff --git a/doc/index.mdx b/doc/index.mdx index d420815d5..a9d127fe2 100644 --- a/doc/index.mdx +++ b/doc/index.mdx @@ -8,14 +8,14 @@ CloudAI benchmark framework aims to develop an industry standard benchmark focus This document contains the following chapters: -- [Tutorial](Tutorial.mdx) -- [User Guide](USER_GUIDE.mdx) -- [Python API](api.mdx) -- [Reporting](reporting.mdx) -- [Systems](systems.mdx) -- [Workloads](workloads/index.mdx) -- [Development](DEV.mdx) -- [Troubleshooting](troubleshooting.mdx) +- [Tutorial](/Tutorial) +- [User Guide](/USER_GUIDE) +- [Python API](/api) +- [Reporting](/reporting) +- [Systems](/systems) +- [Workloads](/workloads/index) +- [Development](/DEV) +- [Troubleshooting](/troubleshooting) @@ -33,7 +33,7 @@ uv run cloudai --help -For more instructions on how to set up access for `enroot`, see [Installation Requirements](workloads/workloads_requirements_installation.mdx). +For more instructions on how to set up access for `enroot`, see [Installation Requirements](/workloads/workloads_requirements_installation). @@ -41,7 +41,7 @@ For more instructions on how to set up access for `enroot`, see [Installation Re ### pip-based Installation -See the preferred Python version in the `.python-version` file and make sure you have it installed (For installation, see [Custom Python Version Installation](index.mdx#install-custom-python-version)). Python 3.10 and newer remain supported. Follow these steps: +See the preferred Python version in the `.python-version` file and make sure you have it installed (For installation, see [Custom Python Version Installation](#install-custom-python-version)). Python 3.10 and newer remain supported. Follow these steps: ``` bash git clone git@github.com:NVIDIA/cloudai.git @@ -96,13 +96,13 @@ The following are the global options for the `cloudai` command: ### Available Modes -- [run](index.mdx#run) -- [dry-run](index.mdx#dry-run) -- [generate-report](index.mdx#generate-report) -- [install](index.mdx#install) -- [uninstall](index.mdx#uninstall) -- [list](index.mdx#list) -- [verify-configs](index.mdx#verify-configs) +- [run](#run) +- [dry-run](#dry-run) +- [generate-report](#generate-report) +- [install](#install) +- [uninstall](#uninstall) +- [list](#list) +- [verify-configs](#verify-configs) @@ -152,7 +152,7 @@ cloudai generate-report\ #### install -This mode installs test prerequisites. For more details, refer to the [Installation Requirements](workloads/workloads_requirements_installation.mdx) guide. It automatically runs as part of the `run` mode if prerequisites are not met. +This mode installs test prerequisites. For more details, refer to the [Installation Requirements](/workloads/workloads_requirements_installation) guide. It automatically runs as part of the `run` mode if prerequisites are not met. ``` bash cloudai install\ diff --git a/doc/pareto-frontiers.mdx b/doc/pareto-frontiers.mdx index a93e05f06..5ee0a4df5 100644 --- a/doc/pareto-frontiers.mdx +++ b/doc/pareto-frontiers.mdx @@ -6,7 +6,7 @@ The `pareto` scenario report writes `pareto.html` in the scenario results direct ## Usage Example -For [vLLM](workloads/vllm.mdx) or [SGLang](workloads/sglang.mdx), add this block to your scenario TOML: +For [vLLM](/workloads/vllm) or [SGLang](/workloads/sglang), add this block to your scenario TOML: ``` toml [reports.pareto] diff --git a/doc/reporting.mdx b/doc/reporting.mdx index eae1936cb..c13cdbd30 100644 --- a/doc/reporting.mdx +++ b/doc/reporting.mdx @@ -2,14 +2,14 @@ This chapter describes the reporting system in CloudAI. In this chapter, we will cover the following topics: -- [Overview](reporting.mdx#overview) -- [General Flow](reporting.mdx#general-flow) -- [Enabling, Disabling and Configuring Reports](reporting.mdx#enabling-disabling-and-configuring-reports) -- [Reporting Registration](reporting.mdx#reporting-registration) -- [Reporting Configuration Implementation](reporting.mdx#reporting-configuration-implementation) -- [Uploading Results to Object Storage](reporting.mdx#uploading-results-to-object-storage) -- [Pareto frontiers](pareto-frontiers.mdx) -- [Reports](reports.mdx) +- [Overview](#overview) +- [General Flow](#general-flow) +- [Enabling, Disabling and Configuring Reports](#enabling-disabling-and-configuring-reports) +- [Reporting Registration](#reporting-registration) +- [Reporting Configuration Implementation](#reporting-configuration-implementation) +- [Uploading Results to Object Storage](#uploading-results-to-object-storage) +- [Pareto frontiers](/pareto-frontiers) +- [Reports](/reports) @@ -22,7 +22,7 @@ CloudAI has two reporting levels: All reports are generated after the test scenario is completed as part of the main CloudAI process. For Slurm, this means that the login node is used to generate reports. -Per-test reports are linked to a particular workload type (e.g. `NcclTest`). All per-test reports are implemented as part of the `per_test` scenario report and can be enabled or disabled via a single configuration option; see [Enabling, Disabling and Configuring Reports](reporting.mdx#enabling-disabling-and-configuring-reports). +Per-test reports are linked to a particular workload type (e.g. `NcclTest`). All per-test reports are implemented as part of the `per_test` scenario report and can be enabled or disabled via a single configuration option; see [Enabling, Disabling and Configuring Reports](#enabling-disabling-and-configuring-reports). To list all available reports, users can use `cloudai list-reports`. Use verbose output to also print report configurations. diff --git a/doc/reports.mdx b/doc/reports.mdx index 504851875..e0f1c7d76 100644 --- a/doc/reports.mdx +++ b/doc/reports.mdx @@ -2,4 +2,4 @@ CloudAI report schemas describe the generated output for supported workload categories. -- [Training Report Schema](training-report-schema.mdx) +- [Training Report Schema](/training-report-schema) diff --git a/doc/systems.mdx b/doc/systems.mdx index 93687e450..6e384500c 100644 --- a/doc/systems.mdx +++ b/doc/systems.mdx @@ -4,11 +4,11 @@ This chapter lists all the systems supported by CloudAI. The attributes shown fo | System | Scheduler Value | |---------------------------------------------|-----------------| -| [Slurm](systems.mdx#slurm-system) | `slurm` | -| [Kubernetes](systems.mdx#kubernetes-system) | `kubernetes` | -| [RunAI](systems.mdx#runai-system) | `runai` | -| [LSF](systems.mdx#lsf-system) | `lsf` | -| [Standalone](systems.mdx#standalone-system) | `standalone` | +| [Slurm](#slurm-system) | `slurm` | +| [Kubernetes](#kubernetes-system) | `kubernetes` | +| [RunAI](#runai-system) | `runai` | +| [LSF](#lsf-system) | `lsf` | +| [Standalone](#standalone-system) | `standalone` | diff --git a/doc/workloads/index.mdx b/doc/workloads/index.mdx index 6597c4b59..23381c86d 100644 --- a/doc/workloads/index.mdx +++ b/doc/workloads/index.mdx @@ -8,30 +8,30 @@ This chapter contains automatically generated documentation for all CloudAI work | Test | Slurm | Kubernetes | RunAI | Standalone | |----|----|----|----|----| -| [AIConfigurator](aiconfigurator.mdx) | ❌ | ❌ | ❌ | ✅ | -| [AI Dynamo](ai_dynamo.mdx) | ✅ | ✅ | ❌ | ❌ | -| [Bash Command](bash_cmd.mdx) | ✅ | ❌ | ❌ | ❌ | -| [Chakra Replay](chakra_replay.mdx) | ✅ | ❌ | ❌ | ❌ | -| [DDLB](ddlb.mdx) | ✅ | ❌ | ❌ | ❌ | -| [DeepEP Benchmark](deepep.mdx) | ✅ | ❌ | ❌ | ❌ | -| [DynamoMocker](dynamo_mocker.mdx) | ❌ | ❌ | ❌ | ✅ | -| [fio](fio.mdx) | ✅ | ❌ | ❌ | ✅ | -| [JaxToolbox workloads (DEPRECATED)](jax_toolbox.mdx) | ✅ | ❌ | ❌ | ❌ | +| [AIConfigurator](/workloads/aiconfigurator) | ❌ | ❌ | ❌ | ✅ | +| [AI Dynamo](/workloads/ai_dynamo) | ✅ | ✅ | ❌ | ❌ | +| [Bash Command](/workloads/bash_cmd) | ✅ | ❌ | ❌ | ❌ | +| [Chakra Replay](/workloads/chakra_replay) | ✅ | ❌ | ❌ | ❌ | +| [DDLB](/workloads/ddlb) | ✅ | ❌ | ❌ | ❌ | +| [DeepEP Benchmark](/workloads/deepep) | ✅ | ❌ | ❌ | ❌ | +| [DynamoMocker](/workloads/dynamo_mocker) | ❌ | ❌ | ❌ | ✅ | +| [fio](/workloads/fio) | ✅ | ❌ | ❌ | ✅ | +| [JaxToolbox workloads (DEPRECATED)](/workloads/jax_toolbox) | ✅ | ❌ | ❌ | ❌ | | MegatronRun | ✅ | ❌ | ❌ | ❌ | -| [MegatronBridge](megatron_bridge.mdx) | ✅ | ❌ | ❌ | ❌ | -| [NCCL](nccl.mdx) | ✅ | ✅ | ✅ | ❌ | -| [NeMo v1.0 aka NemoLauncher (DEPRECATED)](nemo_launcher.mdx) | ✅ | ❌ | ❌ | ❌ | -| [Nemo Run](nemo_run.mdx) | ✅ | ❌ | ❌ | ❌ | -| [NIXL Bench](nixl_bench.mdx) | ✅ | ❌ | ❌ | ❌ | -| [NIXL EP](nixl_ep.mdx) | ✅ | ❌ | ❌ | ❌ | -| [NIXL KVBench](nixl_kvbench.mdx) | ✅ | ❌ | ❌ | ❌ | -| [NIXL CTPerf](nixl_perftest.mdx) | ✅ | ❌ | ❌ | ❌ | -| [Sleep](sleep.mdx) | ✅ | ✅ | ❌ | ✅ | -| [SGLang](sglang.mdx) | ✅ | ❌ | ❌ | ❌ | -| [Slurm Container](slurm_container.mdx) | ✅ | ❌ | ❌ | ❌ | +| [MegatronBridge](/workloads/megatron_bridge) | ✅ | ❌ | ❌ | ❌ | +| [NCCL](/workloads/nccl) | ✅ | ✅ | ✅ | ❌ | +| [NeMo v1.0 aka NemoLauncher (DEPRECATED)](/workloads/nemo_launcher) | ✅ | ❌ | ❌ | ❌ | +| [Nemo Run](/workloads/nemo_run) | ✅ | ❌ | ❌ | ❌ | +| [NIXL Bench](/workloads/nixl_bench) | ✅ | ❌ | ❌ | ❌ | +| [NIXL EP](/workloads/nixl_ep) | ✅ | ❌ | ❌ | ❌ | +| [NIXL KVBench](/workloads/nixl_kvbench) | ✅ | ❌ | ❌ | ❌ | +| [NIXL CTPerf](/workloads/nixl_perftest) | ✅ | ❌ | ❌ | ❌ | +| [Sleep](/workloads/sleep) | ✅ | ✅ | ❌ | ✅ | +| [SGLang](/workloads/sglang) | ✅ | ❌ | ❌ | ❌ | +| [Slurm Container](/workloads/slurm_container) | ✅ | ❌ | ❌ | ❌ | | Triton Inference | ✅ | ❌ | ❌ | ❌ | -| [UCC](ucc.mdx) | ✅ | ❌ | ❌ | ❌ | -| [vLLM](vllm.mdx) | ✅ | ❌ | ❌ | ❌ | +| [UCC](/workloads/ucc) | ✅ | ❌ | ❌ | ❌ | +| [vLLM](/workloads/vllm) | ✅ | ❌ | ❌ | ❌ | diff --git a/doc/workloads/nemo_launcher.mdx b/doc/workloads/nemo_launcher.mdx index 1d73097a3..a8c43cd35 100644 --- a/doc/workloads/nemo_launcher.mdx +++ b/doc/workloads/nemo_launcher.mdx @@ -1,6 +1,6 @@ -This workload is **DEPRECATED**. Please use [Nemo Run](nemo_run.mdx) instead. +This workload is **DEPRECATED**. Please use [Nemo Run](/workloads/nemo_run) instead. From 2748abdb41558cab54ba9d864e3351e674801644 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 30 Oct 2026 14:18:03 +0100 Subject: [PATCH 5/6] Remove redundant example download instruction Signed-off-by: Ivan Podkidyshev --- doc/reporting.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/reporting.mdx b/doc/reporting.mdx index c13cdbd30..a9bd062cb 100644 --- a/doc/reporting.mdx +++ b/doc/reporting.mdx @@ -58,8 +58,6 @@ This example comes from a Slurm NCCL all-reduce run on one node with eight H100 Identifiers and paths are anonymized. To keep the example small, each metrics list includes only out-of-place latency and bus bandwidth for 1 MiB messages. Measurement values, timing, and DSE selection are unchanged. -`Download experiment.json `. - ``` json { "id": "nccl-dse_2026-09-21_14-55-45", From cc71f75cc4f8e0f29f712f3551ab86bb874d5316 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 30 Oct 2026 14:30:17 +0100 Subject: [PATCH 6/6] Check generated dataclass field names against the model Signed-off-by: Ivan Podkidyshev --- tests/doc/test_generate_reference.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/doc/test_generate_reference.py b/tests/doc/test_generate_reference.py index 0bde03ea7..b34f4ca81 100644 --- a/tests/doc/test_generate_reference.py +++ b/tests/doc/test_generate_reference.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import dataclasses import importlib.util from pathlib import Path @@ -85,7 +86,8 @@ def test_dataclass_fields(loader): symbol = "cloudai.report_generator.training.models.TrainingConfig" result = reference.describe_class(symbol, loader.modules_collection[symbol], ["__init__"]) fields = {field["name"]: field for field in result["fields"]} - assert len(fields) == 47 + expected = {field.name for field in dataclasses.fields(reference.load_class(symbol))} + assert set(fields) == expected assert fields["test_id"]["required"] assert fields["container_image"]["default"] == "''" assert all(member["name"] != "__init__" for member in result["members"])