[Docs] Migrate to Fern

.github/workflows/ci.yml

@@ -30,14 +30,6 @@ jobs:
       - name: Run linting via pre-commit
         run: uv run --locked pre-commit run --all-files --show-diff-on-failure
 
-      - name: Build documentation
-        run: |
-          set -eE
-          set -o pipefail
-
-          uv run --locked --extra docs sphinx-build -M html "doc" "doc/_build"
-          touch doc/_build/html/.nojekyll
-
   test:
     name: Run pytest (3.14)
 

.github/workflows/docs.yml

@@ -1,81 +1,41 @@
-name: Build and Deploy Documentation
+name: Validate Fern Documentation
 
 on:
+  pull_request:
   push:
-    branches: [ main ]
-    tags: [ 'v*' ]
-  workflow_dispatch:    # Allow manual triggering
+    branches: [main]
+    tags: ['v*']
+  workflow_dispatch:
 
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
+permissions:
+  contents: read
 
 jobs:
-  build:
+  docs:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout
-      uses: actions/checkout@v6
-
-    - name: Install uv
-      uses: astral-sh/setup-uv@v5
-      with:
-        enable-cache: true
-
-    - name: Build documentation
-      run: make -C doc/ html
-
-    - name: Upload artifact
-      uses: actions/upload-pages-artifact@v4
-      with:
-        path: ./doc/_build/html
-
-  build-cms:
-    name: Build CMS Documentation
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: Install uv
-      uses: astral-sh/setup-uv@v5
-      with:
-        enable-cache: true
-
-    - name: Set up Node.js
-      uses: actions/setup-node@v6
-      with:
-        node-version: '24'
-
-    - name: Build CMS documentation
-      run: make -C doc/ cms-html
-
-    - name: Sanitize artifact name
-      id: artifact-name
-      run: echo "name=cloudai-docs-cms-$(echo '${{ github.ref_name }}' | tr '/' '-')" >> $GITHUB_OUTPUT
-
-    - name: Upload CMS docs artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ steps.artifact-name.outputs.name }}
-        path: doc/_build-cms/html
-
-  deploy:
-    name: Deploy to GitHub Pages
-    needs: build
-
-    permissions:
-      pages: write
-      id-token: write
-
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-
-    runs-on: ubuntu-latest
-    steps:
-    - name: Deploy to GitHub Pages
-      id: deployment
-      uses: actions/deploy-pages@v4
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version-file: .python-version
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '24'
+          cache: npm
+          cache-dependency-path: fern/package-lock.json
+      - name: Generate and validate documentation
+        run: make -C doc check
+      - name: Test reference generation
+        run: uv run --locked --extra dev --extra docs pytest tests/doc/test_generate_reference.py
+      - name: Upload Fern sources
+        uses: actions/upload-artifact@v4
+        with:
+          name: cloudai-fern-docs
+          path: |
+            doc/**/*.mdx
+            fern/docs.yml
+            fern/fern.config.json
+            fern/generated/

.gitignore

@@ -18,7 +18,8 @@ wheels/
 share/python-wheels/
 *.egg-info/
 *.egg
-doc/_build
+fern/generated/
+fern/node_modules/
 
 # PyInstaller
 # Usually these files are written by a python script from a template

CONTRIBUTING.md

@@ -1,6 +1,10 @@
 # CloudAI OSS Contribution Guidelines
 Welcome to the CloudAI project! This document outlines the procedures and standards for contributing to the CloudAI source code. Adherence to these guidelines ensures code quality, readability, and maintainability. We look forward to your contributions!
 
+## Documentation
+
+See [Documentation](doc/README.md) for authoring, validation, and local preview instructions.
+
 ## Contribution Process
 ### Identifying the Feature or Fix
 Contributors looking to add features or fixes are encouraged to reach out to the team by creating issues on the project's issue tracker. This step ensures that efforts are not duplicated and that the proposed changes align with the project's goals. Please communicate with the main developers before starting work to avoid the potential for your PR not being accepted.

doc/DEV.mdx

@@ -0,0 +1,99 @@
+<Anchor id="development" />
+
+This chapter targets developers who want to contribute to the project's core.
+
+``` mermaid
+graph TD
+    subgraph _core
+        base_modules
+        core_implementations
+        registry
+    end
+
+    subgraph runners
+        SlurmRunner
+        StandaloneRunner
+    end
+
+    subgraph installers
+        SlurmInstaller
+        StandaloneInstaller
+    end
+
+    subgraph systems
+        SlurmSystem
+        StandaloneSystem
+    end
+
+    installers --> _core
+    runners --> _core
+    systems --> _core
+```
+
+<Anchor id="core-modules" />
+
+## Core Modules
+
+We use [import-linter](https://github.com/seddonym/import-linter) to ensure no core modules import higher level modules.
+
+`Registry` object is a singleton that holds implementation mappings. Users can register their own implementations to the registry or replace the default implementations.
+
+<Anchor id="cache" />
+
+## Cache
+
+Some prerequisites can be installed. For example:
+
+Docker images, git repos with executable scripts, etc. All such "installables" are kept under the system's `install_path`.
+
+Installables are shared among all tests. Therefore, if any number of tests use the same installable, it is installed only once for a particular system TOML.
+
+`PythonExecutable` installs a Python project into its own virtual environment. When the selected
+`pyproject.toml` has an adjacent `uv.lock`, it uses bundled uv to install the locked base dependencies
+and the project, without extras or dependency groups. An outdated lockfile fails installation instead
+of being updated. Projects without a lockfile continue to use pip. When `dependencies_from_pyproject=False`
+selects an existing `requirements.txt`, that selection takes precedence over the lockfile.
+
+``` mermaid
+classDiagram
+    class Installable {
+        <<abstract>>
+        + __eq__(other: object)
+        + __hash__()
+    }
+
+    class DockerImage {
+        + url: str
+        + install_path: str | Path
+    }
+
+    class GitRepo {
+        + git_url: str
+        + commit_hash: str
+        + install_path: Path
+    }
+
+    class PythonExecutable {
+        + git_repo: GitRepo
+        + venv_path: Path
+    }
+
+    Installable <|-- DockerImage
+    Installable <|-- GitRepo
+    Installable <|-- PythonExecutable
+    PythonExecutable --> GitRepo
+
+    class BaseInstaller {
+        <<abstract>>
+        + install(items: Iterable[Installable])
+        + uninstall(items: Iterable[Installable])
+        + is_installed(items: Iterable[Installable]) -> bool
+
+        * install_one(item: Installable)
+        * uninstall_one(item: Installable)
+        * is_installed_one(item: Installable) -> bool
+    }
+
+    BaseInstaller <|-- SlurmInstaller
+    BaseInstaller <|-- StandaloneInstaller
+```