diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 5d0d6143b..096735816 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,9 +1,6 @@ #!/usr/bin/env bash -# Regenerate the RHDH form, the profile catalogs and the installer's mirror -# of the governance provider profiles when a profile (or the generator) is -# part of the commit, and add them to the commit, so adding a profile is only -# adding its directory under charts/saw-bom/profiles/ (or its file under -# charts/governance-policy/profiles/). +# Regenerate the forms/catalogs and canonical source mirrors when an input +# changes. RHDH and standalone installation package the same workspace projection. # # Enable once per clone with: make git-hooks # (Portable to macOS's bash 3.2: no mapfile, no associative arrays.) @@ -12,7 +9,7 @@ set -euo pipefail # Capture first: `git diff | grep -q` under pipefail can fail with SIGPIPE # when grep stops reading early, which would skip the regeneration. staged=$(git diff --cached --name-only) -if ! grep -qE '^(charts/saw-bom/profiles/|charts/governance-policy/profiles/|scripts/saw-profile-catalog\.py$)' <<<"$staged"; then +if ! grep -qE '^(charts/saw-bom/profiles/|charts/governance-policy/profiles/|charts/openshell-saw/files/provider-profiles/|charts/openshell-saw/files/installer/saw_ui_limit\.py$|charts/saw-workspace-relay/files/saw_ui_limit\.py$|charts/openshell-saw/templates/_governed-proxy-policy\.tpl$|charts/saw-users/files/workspace\.py$|charts/openshell-keycloak/files/saw_keycloak_admin\.py$|scripts/saw-profile-catalog\.py$)' <<<"$staged"; then exit 0 fi diff --git a/.github/workflows/build-governance-interceptor.yml b/.github/workflows/build-governance-interceptor.yml index 26beff805..254f73646 100644 --- a/.github/workflows/build-governance-interceptor.yml +++ b/.github/workflows/build-governance-interceptor.yml @@ -22,7 +22,6 @@ on: permissions: contents: read - packages: write concurrency: group: build-governance-interceptor-${{ github.ref }} @@ -32,7 +31,8 @@ jobs: # One native build per architecture (no QEMU), each pushed by digest; the # merge job joins them into one multi-arch :latest / :sha- tag. build: - strategy: + if: github.event_name == 'pull_request' || github.ref != 'refs/heads/main' + strategy: &platforms # Each architecture reports its own result; merge still needs both. fail-fast: false matrix: @@ -43,7 +43,7 @@ jobs: runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} - steps: + steps: &build_steps - uses: actions/checkout@v4 - name: Read build parameters from values.yaml @@ -78,7 +78,7 @@ jobs: # the interceptor must match the SAW gateways' OpenShell release. extra_tag: ${{ steps.params.outputs.openshell_ref }} platform: ${{ matrix.platform }} - registry_token: ${{ secrets.GITHUB_TOKEN }} + registry_token: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && secrets.GITHUB_TOKEN || '' }} # Compile the OpenShift build source too, including its behavioral # tests, on each architecture. This targets the builder stage; no @@ -120,8 +120,16 @@ jobs: cache-to: type=gha,mode=max,scope=governance-interceptor-openshift-${{ matrix.runner }},ignore-error=true provenance: false + publish: + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + permissions: {contents: read, packages: write} + strategy: *platforms + runs-on: ${{ matrix.runner }} + steps: *build_steps + merge: - needs: build + needs: publish + permissions: {contents: read, packages: write} if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/build-kubernetes-runtime.yml b/.github/workflows/build-kubernetes-runtime.yml new file mode 100644 index 000000000..36a45f813 --- /dev/null +++ b/.github/workflows/build-kubernetes-runtime.yml @@ -0,0 +1,79 @@ +name: Build Kubernetes workspace runtime image + +on: + push: + branches: [main] + paths: + - kubernetes-runtime/** + - charts/openshell-saw/files/installer/saw_core.py + - charts/openshell-saw/files/installer/saw_admin.py + - charts/openshell-saw/files/installer/tool-*.mjs + - charts/openshell-saw/files/installer/tool-actions.yaml + - charts/openshell-saw/files/provider-profiles/** + - charts/saw-users/files/workspace.py + - .github/workflows/build-kubernetes-runtime.yml + - .github/actions/build-push-image/** + - .github/actions/merge-image/** + - .github/actions/verify-public-image/** + pull_request: + paths: + - kubernetes-runtime/** + - charts/openshell-saw/files/installer/saw_core.py + - charts/openshell-saw/files/installer/saw_admin.py + - charts/openshell-saw/files/installer/tool-*.mjs + - charts/openshell-saw/files/installer/tool-actions.yaml + - charts/openshell-saw/files/provider-profiles/** + - charts/saw-users/files/workspace.py + - .github/workflows/build-kubernetes-runtime.yml + - .github/actions/build-push-image/** + - .github/actions/merge-image/** + - .github/actions/verify-public-image/** + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: build-kubernetes-runtime-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + if: github.event_name == 'pull_request' || github.ref != 'refs/heads/main' + strategy: &platforms + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: &build_steps + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-image + with: + image_name: saw-kubernetes-runtime + context: . + dockerfile: kubernetes-runtime/Containerfile + cache_scope: saw-kubernetes-runtime + platform: ${{ matrix.platform }} + registry_token: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && secrets.GITHUB_TOKEN || '' }} + publish: + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + permissions: {contents: read, packages: write} + strategy: *platforms + runs-on: ${{ matrix.runner }} + steps: *build_steps + + merge: + needs: publish + permissions: {contents: read, packages: write} + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/merge-image + with: + image_name: saw-kubernetes-runtime + registry_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/installer-tests.yml b/.github/workflows/installer-tests.yml index 13dd4a273..755bcbbb4 100644 --- a/.github/workflows/installer-tests.yml +++ b/.github/workflows/installer-tests.yml @@ -59,6 +59,27 @@ jobs: mkdir -p test-results set -o pipefail make lint-static 2>&1 | tee test-results/lint.log + - name: Compiler and governance admission contract + env: + SAW_REQUIRE_RUST_TESTS: "1" + run: | + set -euo pipefail + cargo --version + scratch=$(mktemp -d) + trap 'rm -rf "$scratch"' EXIT + mkdir "$scratch/src" + touch "$scratch/src/lib.rs" + cat > "$scratch/Cargo.toml" <<'EOF' + [package] + name = "saw-guard-test-dependencies" + version = "0.1.0" + edition = "2021" + [dependencies] + serde_json = "1" + EOF + cargo fetch --manifest-path "$scratch/Cargo.toml" + python -m pytest -q tests/kubernetes_runtime/test_governance_contract.py \ + tests/charts/test_governance_interceptor_image.py::test_exact_embedded_guard_behavior - name: External tool and static asset tests run: | mkdir -p test-results @@ -107,6 +128,24 @@ jobs: version: v3.17.3 - name: Install test dependencies run: python -m pip install -r tests/requirements.txt -e ./cli + - name: Check out the pinned stock OpenShell SDK + if: matrix.python == '3.12' + uses: actions/checkout@v4 + with: + repository: opendatahub-io/openshell + ref: 305c8b3d7bbdb97a6f2f7264c3ad43b0087c307f + path: .stock-openshell + persist-credentials: false + sparse-checkout: | + proto + python/openshell + tasks/scripts + - name: Stock protobuf wire and runtime recovery contracts + if: matrix.python == '3.12' + run: | + python -m pip install grpcio-tools==1.80.0 grpcio==1.80.0 protobuf==6.33.5 googleapis-common-protos==1.76.0 httpx==0.28.1 + mv .stock-openshell "$RUNNER_TEMP/saw-stock-openshell" + python kubernetes-runtime/test-stock-protobuf.py --openshell-source "$RUNNER_TEMP/saw-stock-openshell" - name: Python behavior and integration tests run: | mkdir -p test-results diff --git a/.github/workflows/profile-catalog.yml b/.github/workflows/profile-catalog.yml index ac6cc24d6..b7687dc74 100644 --- a/.github/workflows/profile-catalog.yml +++ b/.github/workflows/profile-catalog.yml @@ -25,16 +25,34 @@ on: paths: - "charts/saw-bom/profiles/**" - "charts/governance-policy/profiles/**" + - "charts/openshell-saw/templates/_governed-proxy-policy.tpl" - "charts/openshell-saw/files/provider-profiles/**" + - "charts/openshell-saw/files/installer/saw_ui_limit.py" + - "charts/saw-workspace-relay/files/saw_ui_limit.py" + - "charts/openshell-keycloak/files/saw_keycloak_admin.py" + - "charts/openshell-rhdh/files/saw_keycloak_admin.py" + - "charts/saw-users/files/workspace.py" + - "charts/openshell-rhdh/files/workspace.py" - "scripts/saw-profile-catalog.py" - "charts/saw-users/files/profile-catalog.json" + - "charts/saw-users/files/governed-profiles.json" + - "charts/saw-users/templates/_governed-proxy-policy.tpl" - "charts/openshell-rhdh/files/profile-catalog.json" - "charts/openshell-rhdh/files/create-workspace*.yaml" - "charts/openshell-rhdh/files/delete-workspace*.yaml" push: branches: [main] paths: + - "charts/governance-policy/profiles/**" + - "charts/openshell-saw/files/provider-profiles/**" + - "charts/openshell-saw/files/installer/saw_ui_limit.py" + - "charts/saw-workspace-relay/files/saw_ui_limit.py" + - "charts/openshell-saw/templates/_governed-proxy-policy.tpl" - "charts/saw-bom/profiles/**" + - "charts/openshell-keycloak/files/saw_keycloak_admin.py" + - "charts/openshell-rhdh/files/saw_keycloak_admin.py" + - "charts/saw-users/files/workspace.py" + - "charts/openshell-rhdh/files/workspace.py" - "scripts/saw-profile-catalog.py" workflow_dispatch: diff --git a/Makefile b/Makefile index ae4f97097..7405b836e 100644 --- a/Makefile +++ b/Makefile @@ -9,3 +9,10 @@ include Makefile-quickstart # the VM disks (they set no storageClassName). Interactive on a TTY; offers to # reuse an existing class or enable HPP. See scripts/storage-class-check.sh. pattern-install: storage-class-check + +# Bootstrap stock GitOps before the framework selects its Argo CD namespace. +.PHONY: prepare-gitops +prepare-gitops: + @$(ANSIBLE_RUN) ansible/playbooks/saw-gitops-bootstrap.yaml + +pattern-install operator-deploy: prepare-gitops diff --git a/Makefile-quickstart b/Makefile-quickstart index 7dfd513d3..af02aefbb 100644 --- a/Makefile-quickstart +++ b/Makefile-quickstart @@ -19,6 +19,10 @@ IMAGE_TAG ?= latest # OpenShell release the governance interceptor is built from (must match the # SAW gateways; image-builder-charts/helm/governance-interceptor-image). OPENSHELL_INTERCEPTOR_REF ?= v0.1.6 +DEPLOYMENT_MODE ?= vm +SAW_USERS_VALUES_FILE ?= +SAW_INSTALL_NAMESPACE ?= saw-workspaces +export DEPLOYMENT_MODE SAW_USERS_VALUES_FILE SAW_INSTALL_NAMESPACE OPENSHELL_SAW_NAME ?= SANDBOX_IMAGE ?= CONTAINER_RUNTIME ?= podman @@ -142,6 +146,11 @@ check-oidc-ca: ## Check if the OIDC issuer needs a CA bundle ssh-key-generate: ## Generate or recover the SSH public key without replacing the private key @$(SCRIPTS_DIR)/generate-keys.sh +# Build once per architecture before enabling the container data-science profile. +.PHONY: container-notebook-image +container-notebook-image: ## Build the notebook image for OpenShift namespace UIDs and print the public site values + @$(SCRIPTS_DIR)/container-notebook-image.sh + # --- NemoClaw images --- sandbox-build: .check-image-repo ## Build audited discovery artifacts, then the sandbox (IMAGE_REPO opts into pushing) @@ -299,9 +308,10 @@ clean-stale-operators: ## Inspect stale CNV CRDs after operator cleanup # CSV/InstallPlan) is intentionally deferred to the uninstall target below, # after the Ansible framework removes ArgoCD apps — otherwise ArgoCD # immediately recreates the Subscription we just deleted. -pre-uninstall: ## Delete pattern VMs before framework uninstall +pre-uninstall: ## Drain GitOps writers, then delete owned VMs before framework uninstall @$(SCRIPTS_DIR)/pattern-cnv-mark.sh @$(SCRIPTS_DIR)/pattern-pipelines-mark.sh + @python3 $(SCRIPTS_DIR)/pattern-drain-applications.py @$(SCRIPTS_DIR)/pattern-pre-uninstall.sh # Generate SSH keys before install. No deletions — safe to run repeatedly. @@ -327,8 +337,8 @@ pattern-uninstall: uninstall ## Uninstall the Validated Pattern # Clean up one sandbox's VM and DataVolume before saw-delete. This target # handles only OPENSHELL_SAW_NAME; pattern uninstall has its own cleanup. -sandbox-pre-delete: .check-saw-name ## Schedule deletion of a sandbox's VM and disk - @$(SCRIPTS_DIR)/sandbox-pre-delete.sh +sandbox-pre-delete: .check-saw-name .check-deployment-mode ## Schedule deletion of a sandbox's VM and disk + @if [ "$(DEPLOYMENT_MODE)" = vm ]; then $(SCRIPTS_DIR)/sandbox-pre-delete.sh; fi # --- Keycloak / OIDC --- @@ -412,8 +422,25 @@ keycloak-delete: ## Uninstall Keycloak and its database ##@ Sandbox Operations -saw-create: .check-saw-name ## Provision one SAW (OPENSHELL_SAW_NAME is required) - @$(SCRIPTS_DIR)/openshell-saw-create.sh +.PHONY: container-topology check-container-topology +container-topology: ## Print the site network overlay discovered from OpenShift and a trusted Pod + @$(or $(PYTHON),uv run --locked python) $(SCRIPTS_DIR)/container_topology.py + +check-container-topology: ## Compare the reviewed site network overlay with live OpenShift observations + @test -n "$${SAW_TOPOLOGY_OVERLAY}" || { echo "Set SAW_TOPOLOGY_OVERLAY" >&2; exit 1; } + @$(or $(PYTHON),uv run --locked python) $(SCRIPTS_DIR)/container_topology.py --check "$${SAW_TOPOLOGY_OVERLAY}" + +saw-create: .check-saw-name .check-deployment-mode ## Provision one SAW using DEPLOYMENT_MODE=vm, container or kata + @if [ "$(DEPLOYMENT_MODE)" != vm ]; then \ + $(or $(PYTHON),uv run --locked python) $(SCRIPTS_DIR)/container-workspace.py apply; \ + else $(SCRIPTS_DIR)/openshell-saw-create.sh; fi + +.PHONY: saw-preflight +saw-preflight: .check-saw-name .check-deployment-mode ## Check Kubernetes workspace prerequisites without changing the cluster + @if [ "$(DEPLOYMENT_MODE)" = vm ]; then \ + echo "saw-preflight applies to container and kata modes" >&2; exit 1; \ + fi + @$(or $(PYTHON),uv run --locked python) $(SCRIPTS_DIR)/container-workspace.py preflight saw-configure: .check-saw-name ## Configure the OpenShell CLI for one gateway @$(SCRIPTS_DIR)/saw-configure.sh @@ -423,15 +450,19 @@ saw-configure: .check-saw-name ## Configure the OpenShell CLI for one gateway saw-list: ## List SAW virtual machines across namespaces @$(SCRIPTS_DIR)/saw-list.sh -saw-logs: .check-saw-name ## Follow the VM installer log +saw-logs: .check-saw-name .check-vm-deployment ## Follow the VM installer log oc -n "$(SAW_NS)" logs $(if $(filter false,$(FOLLOW)),,-f) \ -l "vm.kubevirt.io/name=$(OPENSHELL_SAW_NAME)" -c guest-console-log --tail=-1 -saw-status: .check-saw-name .check-ssh-key ## Show both installer phases from the VM - @VM_NAME="$(OPENSHELL_SAW_NAME)" \ - $(SCRIPTS_DIR)/openshell-saw-vm-ssh.sh 'sudo cat /var/lib/saw/status.json' +saw-status: .check-saw-name .check-deployment-mode ## Show the selected workspace's installation status + @if [ "$(DEPLOYMENT_MODE)" != vm ]; then \ + $(or $(PYTHON),uv run --locked python) $(SCRIPTS_DIR)/container-workspace.py status; \ + else \ + $(MAKE) .check-ssh-key && VM_NAME="$(OPENSHELL_SAW_NAME)" \ + $(SCRIPTS_DIR)/openshell-saw-vm-ssh.sh 'sudo cat /var/lib/saw/status.json'; \ + fi -saw-vm-ssh: .check-saw-name .check-ssh-key ## Open a VM shell (CMD= runs one command) +saw-vm-ssh: .check-saw-name .check-vm-deployment .check-ssh-key ## Open a VM shell (CMD= runs one command) @VM_NAME="$(OPENSHELL_SAW_NAME)" \ $(SCRIPTS_DIR)/openshell-saw-vm-ssh.sh @@ -449,11 +480,11 @@ harness-bundle-push: ## Push harness-bundles/$(HARNESS_BUNDLE) to $(HARNESS_REPO @test -n "$(HARNESS_REPO)" || { echo "set HARNESS_REPO, e.g. ghcr.io//saw-harness-$(HARNESS_BUNDLE)"; exit 1; } @$(SCRIPTS_DIR)/harness-bundle.sh push $(HARNESS_BUNDLE) $(HARNESS_REPO) -saw-restart: .check-saw-name .check-ssh-key ## Restart through guest poweroff to apply changed inputs +saw-restart: .check-saw-name .check-vm-deployment .check-ssh-key ## Restart through guest poweroff to apply changed inputs @VM_NAME="$(OPENSHELL_SAW_NAME)" \ $(SCRIPTS_DIR)/openshell-saw-restart.sh -saw-restart-pending: ## Restart every SAW VM waiting for a restart (CONFIRM=yes; lists them otherwise) +saw-restart-pending: .check-vm-deployment ## Restart every SAW VM waiting for a restart (CONFIRM=yes; lists them otherwise) @CONFIRM=$(CONFIRM) $(SCRIPTS_DIR)/saw-restart-pending.sh saw-ssh: .check-saw-name .check-gateway-ca ## Open a shell in a sandbox @@ -575,14 +606,37 @@ test-pattern-e2e-cleanup: ## Clean an E2E run when ready (set CI_RECOVERY_RUN_ID .PHONY: lint-static lint-interface lint: lint-static lint-interface ## Lint charts, shell syntax, and the Make interface +# Helm lint softens template fail() guards and has no --api-versions flag. +# Runtime charts also need enabled template renders with synthetic discovery; +# these check renderability only, never API availability on a real cluster. lint-static: ## Lint charts and shell syntax without running pytest @for chart in charts/* image-builder-charts/helm/*; do \ test -f "$$chart/Chart.yaml" || continue; \ if [ "$$chart" = charts/openshell-saw ]; then \ helm lint "$$chart" --set sandboxName=lint -f tests/fixtures/saw-network-topology.yaml || exit; \ helm lint "$$chart" --set sandboxName=lint -f tests/fixtures/saw-network-topology.yaml --set credentialProxy.enabled=false || exit; \ + elif [ "$$chart" = charts/openshell-saw-kubernetes ]; then \ + helm lint "$$chart" --namespace saw-lint -f tests/fixtures/saw-kubernetes-values.yaml || exit; \ + helm template saw-lint "$$chart" --namespace saw-lint \ + -f tests/fixtures/saw-kubernetes-values.yaml \ + --api-versions agents.x-k8s.io/v1beta1/Sandbox \ + --api-versions policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy >/dev/null || exit; \ + elif [ "$$chart" = charts/saw-kubernetes-identity ]; then \ + helm lint "$$chart" || exit; \ + helm lint "$$chart" --namespace saw-lint-trusted -f tests/fixtures/saw-kubernetes-identity-values.yaml || exit; \ + helm template identity-lint "$$chart" --namespace saw-lint-trusted -f tests/fixtures/saw-kubernetes-identity-values.yaml >/dev/null || exit; \ + elif [ "$$chart" = charts/agent-sandbox ]; then \ + helm lint "$$chart" || exit; \ + helm lint "$$chart" --namespace agent-sandbox-system --set enabled=true || exit; \ + helm template agent-sandbox "$$chart" --namespace agent-sandbox-system --set enabled=true >/dev/null || exit; \ + elif [ "$$chart" = charts/saw-runtime-policies ]; then \ + helm lint "$$chart" || exit; \ + helm lint "$$chart" -f "$$chart/tests/container-values.yaml" || exit; \ + helm template runtime-lint "$$chart" -f "$$chart/tests/container-values.yaml" \ + --api-versions policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy \ + --api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy >/dev/null || exit; \ else \ - helm lint "$$chart" --set sandboxName=lint || exit; \ + helm lint "$$chart" || exit; \ fi; \ done @for script in scripts/*.sh tests/*.sh charts/openshell-saw/files/*.sh \ @@ -591,7 +645,7 @@ lint-static: ## Lint charts and shell syntax without running pytest bash -n "$$script" || exit; \ done @if [ "$${CI:-}" = true ]; then \ - python scripts/lint-rendered-shell.py || exit; \ + $(or $(PYTHON),python) scripts/lint-rendered-shell.py || exit; \ else \ uv run --locked scripts/lint-rendered-shell.py || exit; \ fi @@ -611,7 +665,17 @@ test-harness-e2e: .check-saw-name ## Verify one SAW's harness bundle on the clus # --- Teardown --- saw-delete: sandbox-pre-delete ## Uninstall one sandbox and its owned namespace - @$(SCRIPTS_DIR)/saw-delete.sh + @if [ "$(DEPLOYMENT_MODE)" != vm ]; then \ + $(or $(PYTHON),uv run --locked python) $(SCRIPTS_DIR)/container-workspace.py delete; \ + else $(SCRIPTS_DIR)/saw-delete.sh; fi + +.PHONY: .check-deployment-mode +.check-deployment-mode: + @case "$(DEPLOYMENT_MODE)" in vm|container|kata) ;; *) echo "DEPLOYMENT_MODE must be vm, container or kata" >&2; exit 1 ;; esac + +.PHONY: .check-vm-deployment +.check-vm-deployment: .check-deployment-mode + @test "$(DEPLOYMENT_MODE)" = vm || { echo "This command operates on VMs. For container lifecycle and logs, see docs/container-installation.md." >&2; exit 1; } quickstart-delete: ## Remove quickstart Keycloak and image releases @$(SCRIPTS_DIR)/quickstart-delete.sh @@ -624,7 +688,7 @@ status: ## Show status of all OpenShell resources (every SAW namespace + $(NS)) @$(SCRIPTS_DIR)/saw-cluster-status.sh # --- Local Python tests (no cluster needed) --- -TEST_PATHS ?= tests/installer tests/charts tests/scripts tests/portal tests/credential_proxy cli/tests +TEST_PATHS ?= tests/installer tests/charts tests/scripts tests/portal tests/credential_proxy tests/kubernetes_runtime cli/tests PYTEST_ARGS ?= .PHONY: test-installer test-tool-gate test-credential-proxy credential-proxy-image test-credential-isolation GO ?= go @@ -745,3 +809,7 @@ $(eval $(call deprecated_alias,governance-create-profile,governance-profile-crea .PHONY: .check-image-repo .check-image-repo: @$(SCRIPTS_DIR)/check-build-repo.sh "$$PUSH_IMAGE_REPO" + +.PHONY: kata-worker-check +kata-worker-check: ## Report firmware and verify KVM on selected local-Kata workers + @$(or $(PYTHON),uv run --locked python) scripts/kata_worker_check.py diff --git a/README.md b/README.md index 3b8d0c0b2..1372424c8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # Secure Agent Workspace -Deploy isolated, per-user AI agent sandboxes on OpenShift Virtualization with OIDC authentication and policy-controlled access. +Deploy isolated, per-user AI agent sandboxes on OpenShift with OIDC authentication and policy-controlled access. VM is the default backend; container workspaces use the stock OpenShell Kubernetes driver. + +For container mode, see [architecture and data flows](docs/container-workspaces.md) +and [installation, inputs and enablement](docs/container-installation.md). +The VM procedures below remain the default. Container Make installations use +the shared GitOps services described in their installation guide. **Existing SAW deployments using OpenShell older than 0.1.6 must deploy SAW fresh.** In-place upgrades from those versions are not supported. Back up the @@ -214,8 +219,21 @@ https://github.com/user-attachments/assets/6858c0b1-239a-4a6a-b0e4-5872796dcc32 #### Option A: Validated Pattern (automated, GitOps) +Run `./pattern.sh` from a host that can execute the Validated Patterns utility +image natively. For a `linux/amd64` utility image, use an x86_64 Linux bastion +with Podman and the target cluster's kubeconfig. An Apple Silicon host can +manage the cluster with `oc` while the install command runs on that bastion. + Deploys the platform — operators, Vault, ESO, Keycloak, and secrets — via Argo CD. No workspace VM is created until you add a user in step 7 or request a workspace through the portal. +On a fresh cluster, the Make target installs the Red Hat OpenShift GitOps +Operator through OLM and waits for its deployment before creating the Pattern. +It uses the channel and catalog from the framework's rendered configuration. +This prevents a temporary default Argo CD instance from receiving a second +root Application. Existing GitOps installations keep their operator configuration. +The framework's normal sync and health checks run at the end. +[OpenShift GitOps default-instance configuration](https://docs.redhat.com/en/documentation/red_hat_openshift_gitops/1.21/html/argo_cd_instance/setting-up-argocd-instance). + ```bash # 1. Fork, then clone your fork. Argo CD deploys from the branch you push # (users, profiles, endpoints are files in this repository), so GitOps @@ -345,6 +363,8 @@ make images-mirror # 7. Deploy Keycloak (if one is already running in KEYCLOAK_NS, it is used; # you are asked before the OpenShell realm is imported into it; # set USE_EXISTING_KEYCLOAK=yes to import the realm without the prompt) +# A fresh deploy waits for both the server and the openshell-keycloak-realm +# import. The OIDC endpoints are needed by the next two commands. make keycloak-deploy # 8. Verify Keycloak (realm, openshell-cli client, roles) @@ -845,6 +865,15 @@ selection commands, CI coverage, and fixture rules. `make test` and ### Delete +For a clean reinstall, retire workspaces while the portal, Keycloak and Argo +are available. Use the RHDH **Delete my agent workspace** template for portal owners, +remove static owners through Git with `pruneOnRemove: true`, and run +`make saw-delete` with the original site inputs for standalone installations. +Wait for each workspace Application and runtime namespace to disappear before +uninstalling the platform. RHDH preserves generated Applications when its +ApplicationSet disappears. A successful platform uninstall alone can therefore +leave a workspace running. Container audit history is retained separately. + ```bash # Delete a single sandbox make saw-delete @@ -859,6 +888,83 @@ make quickstart-delete ./pattern.sh make pattern-uninstall ``` +A full SAW uninstall first drains the Pattern-owned `openshell-rhdh` and +`saw-users` Applications. The command sets their clustergroup `disabled` +overrides on the Pattern and waits for Argo to mark each child for pruning. +It terminates any active sync and waits for foreground deletion before +the framework removes operators and webhooks. The VM cleanup runs after this +drain so a reconciling SAW Application cannot recreate a VM during teardown. +If the drain fails, correct +the reported problem and rerun `pattern-uninstall`. Those two overrides +persist on the Pattern until uninstall completes. + +To qualify a fresh RHDH-enabled installation, run the first check on the +clean cluster and the second after installing with the configured site inputs: + +```bash +scripts/check-rhdh-bootstrap.sh before +./pattern.sh make install +scripts/check-rhdh-bootstrap.sh after +``` + +ESO creates `rhdh/rhdh-oidc` from Vault before the Keycloak client hook can +start. The hook requires that Secret and has a 15-minute execution deadline. +The check requires the completed Job, a Synced/Healthy RHDH Application, and +a ready rollout of the actual Backstage Deployment. On a cold start, dynamic +plugins can still be installing when Argo reports Healthy. The portal Deployment +has a 30-minute rollout deadline and retains the operator’s startup health +endpoint with a larger failure budget for a cold backend. The check waits up +to 30 minutes for Deployment availability. Set `RHDH_NAME` if the Backstage +resource has a site-specific name. Verify owner sign-in and workspace creation +through the portal separately. After changing owner enrollment, wait until the +RHDH catalog offers the intended deployment mode and profile before submitting +a create request. Catalog ingestion can finish after the Deployment rollout. + +On busy clusters, use a site value to reserve CPU for the framework's +imperative Jobs during cold bootstrap. For example, add this to +`values-cluster-.yaml` and size the requests for available nodes: + +```yaml +clusterGroup: + extraObjects: + bootstrap-resources: + apiVersion: v1 + kind: LimitRange + metadata: + name: bootstrap-requests + namespace: imperative + spec: + limits: + - type: Container + defaultRequest: + cpu: "1" + memory: 256Mi +``` + +Check `oc -n imperative get pods,limitrange` and the admitted unseal Pod's +`spec.initContainers[].resources.requests`. In clustergroup 0.9.58 the stock +unseal playbook has a 600-second timeout. Exit 124 followed by HTTP 403 +creating the Vault secrets engine can indicate an interruption after unseal +and before the stock CLI login. In that state the RHDH ExternalSecret stays +unready. A CPU reservation can reduce contention. It cannot recover an +already interrupted bootstrap. [Stock Job](https://github.com/validatedpatterns/clustergroup-chart/blob/v0.9.58/templates/imperative/unsealjob.yaml), +[stock Vault role](https://github.com/validatedpatterns/rhvp.cluster_utils/blob/6aec8c6d50/roles/vault_utils/tasks/vault_unseal.yaml). + +A reinstall also depends on OpenShift Virtualization cleanup. Uninstall waits +up to one hour for the Pattern-owned `openshift-cnv` namespace to disappear +before checking whether its marked HyperConverged CRD is unused. A timeout +returns an error. Check namespace termination, then rerun +`./pattern.sh make pattern-uninstall` before installing again. A retained +CRD can point OLM at a conversion webhook that no longer exists. + +When switching from an OperatorHub Quickstart to a clean Pattern test, confirm +that both the test CNV namespace and its HyperConverged CRD are absent. A CSV +uninstall can leave the CRD behind. On Pattern teardown, the cleanup command +reads HyperConverged objects through the CRD's served storage version. This +allows it to remove the tracked SAW instance even after the conversion service +disappears. It checks for other instances and subscriptions before deleting +the marked CRD. + ## Repository structure ``` diff --git a/ansible/playbooks/saw-gitops-bootstrap.yaml b/ansible/playbooks/saw-gitops-bootstrap.yaml new file mode 100644 index 000000000..d609eeed5 --- /dev/null +++ b/ansible/playbooks/saw-gitops-bootstrap.yaml @@ -0,0 +1,155 @@ +--- +# Finish OLM bootstrap before the Patterns Operator decides whether this is +# a new cluster or an upgrade using an existing openshift-gitops instance. +- name: Prepare Red Hat OpenShift GitOps for a fresh pattern installation + hosts: localhost + connection: local + gather_facts: false + vars: + include_crds: false + gitops_package: openshift-gitops-operator + gitops_namespace: openshift-gitops-operator + tasks: + - name: Discover existing GitOps subscriptions + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: Subscription + register: subscriptions + + - name: Leave existing operator installations under their current management + ansible.builtin.meta: end_play + when: >- + subscriptions.resources | selectattr('spec.name', 'equalto', gitops_package) + | rejectattr('metadata.labels.saw_gitops_bootstrap', 'defined') | list | length > 0 + + - name: Resolve the same operator catalog and channel as the framework + block: + - ansible.builtin.include_role: + name: rhvp.cluster_utils.pattern_settings + - ansible.builtin.include_role: + name: rhvp.cluster_utils.install_settings + - ansible.builtin.include_role: + name: rhvp.cluster_utils.pattern_install_template + - name: Read the Patterns Operator settings from the rendered installation chart + ansible.builtin.set_fact: + operator_settings: >- + {{ pattern_install_rendered_yaml | from_yaml_all | select('mapping') + | selectattr('kind', 'equalto', 'ConfigMap') + | selectattr('metadata.name', 'equalto', 'patterns-operator-config') | first }} + + - name: Respect an externally managed Patterns Operator or legacy installation namespace + ansible.builtin.meta: end_play + when: >- + operator_settings.metadata.namespace == 'openshift-operators' or + not (pattern_install_rendered_yaml | from_yaml_all | select('mapping') + | selectattr('kind', 'equalto', 'Subscription') + | selectattr('metadata.name', 'equalto', 'patterns-operator') | list) + + - name: Validate the installation target before provisioning the operator + ansible.builtin.include_role: + name: rhvp.cluster_utils.validate_ocp_version + + - name: Check the proposed operator namespace + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ gitops_namespace }}" + register: target_namespace + + - name: Create the operator namespace on a fresh cluster + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ gitops_namespace }}" + when: not target_namespace.resources + + - name: Discover the installed OperatorGroup + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1 + kind: OperatorGroup + namespace: "{{ gitops_namespace }}" + register: operator_groups + + - name: Require a single all-namespaces OperatorGroup when one already exists + ansible.builtin.assert: + that: + - operator_groups.resources | length <= 1 + - not operator_groups.resources or not operator_groups.resources[0].spec.targetNamespaces | default([]) + - not operator_groups.resources or not operator_groups.resources[0].spec.selector | default({}) + + - name: Install the standard all-namespaces OperatorGroup + kubernetes.core.k8s: + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + namespace: "{{ gitops_namespace }}" + name: "{{ gitops_package }}" + spec: {} + when: not operator_groups.resources + + - name: Subscribe to the stock operator without starting the default Argo CD instance + kubernetes.core.k8s: + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + namespace: "{{ gitops_namespace }}" + name: "{{ gitops_package }}" + labels: + saw_gitops_bootstrap: "true" + spec: + name: "{{ gitops_package }}" + channel: "{{ operator_settings.data['gitops.channel'] }}" + source: "{{ operator_settings.data['gitops.catalogSource'] }}" + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic + config: + env: + - {name: ARGOCD_CLUSTER_CONFIG_NAMESPACES, value: "*"} + - {name: DISABLE_DEFAULT_ARGOCD_INSTANCE, value: "true"} + when: not subscriptions.resources | selectattr('spec.name', 'equalto', gitops_package) | list + + - name: Wait for OLM to finish installing GitOps + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: Subscription + namespace: "{{ gitops_namespace }}" + name: "{{ gitops_package }}" + register: subscription + until: >- + subscription.resources | length > 0 and subscription.resources[0].status.installedCSV | default('') | length > 0 + and subscription.resources[0].status.state | default('') == 'AtLatestKnown' + retries: 90 + delay: 10 + + - name: Wait for the operator rollout that disables the default instance + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + namespace: "{{ gitops_namespace }}" + name: openshift-gitops-operator-controller-manager + register: rollout + until: >- + rollout.resources | length > 0 and rollout.resources[0].status.observedGeneration | default(0) == rollout.resources[0].metadata.generation + and rollout.resources[0].status.updatedReplicas | default(0) == rollout.resources[0].spec.replicas + and rollout.resources[0].status.availableReplicas | default(0) == rollout.resources[0].spec.replicas + and rollout.resources[0].status.replicas | default(0) == rollout.resources[0].spec.replicas + and rollout.resources[0].spec.replicas > 0 + and (rollout.resources[0].spec.template.spec.containers | selectattr('name', 'equalto', 'manager') | first).env + | selectattr('name', 'equalto', 'DISABLE_DEFAULT_ARGOCD_INSTANCE') | selectattr('value', 'equalto', 'true') | list | length == 1 + retries: 90 + delay: 10 + + - name: Wait until the operator removes any transient default Argo CD instance + kubernetes.core.k8s_info: + api_version: argoproj.io/v1beta1 + kind: ArgoCD + namespace: openshift-gitops + name: openshift-gitops + register: default_instance + until: default_instance.api_found and not default_instance.resources + retries: 90 + delay: 10 diff --git a/charts/agent-sandbox/Chart.yaml b/charts/agent-sandbox/Chart.yaml new file mode 100644 index 000000000..321344d8b --- /dev/null +++ b/charts/agent-sandbox/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: agent-sandbox +description: Optional platform installation of the unmodified upstream Agent Sandbox controller +type: application +version: 0.1.0 +appVersion: "1.0.9" diff --git a/charts/agent-sandbox/files/LICENSE b/charts/agent-sandbox/files/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/charts/agent-sandbox/files/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/charts/agent-sandbox/files/sandbox.yaml b/charts/agent-sandbox/files/sandbox.yaml new file mode 100644 index 000000000..0c5739faa --- /dev/null +++ b/charts/agent-sandbox/files/sandbox.yaml @@ -0,0 +1,4333 @@ +--- +kind: Namespace +apiVersion: v1 +metadata: + name: agent-sandbox-system + +--- + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: agent-sandbox-controller + namespace: agent-sandbox-system + labels: + app: agent-sandbox-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: agent-sandbox-controller +subjects: +- kind: ServiceAccount + name: agent-sandbox-controller + namespace: agent-sandbox-system +roleRef: + kind: ClusterRole + name: agent-sandbox-controller + apiGroup: rbac.authorization.k8s.io + +--- + +kind: Service +apiVersion: v1 +metadata: + name: agent-sandbox-controller + namespace: agent-sandbox-system + labels: + app: agent-sandbox-controller +spec: + selector: + app: agent-sandbox-controller + ports: + - name: metrics + port: 8080 # Change to 8443 when --metrics-secure-serving is enabled. + targetPort: metrics + protocol: TCP + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + name: agent-sandbox-controller + namespace: agent-sandbox-system + labels: + app: agent-sandbox-controller +spec: + replicas: 1 + selector: + matchLabels: + app: agent-sandbox-controller + template: + metadata: + labels: + app: agent-sandbox-controller + spec: + serviceAccountName: agent-sandbox-controller + containers: + - name: agent-sandbox-controller + image: registry.k8s.io/agent-sandbox/agent-sandbox-controller:v1.0.9 + args: + - --leader-elect=true + # Uncomment to serve metrics over HTTPS with a TLS profile. + # When enabling, also change the Service port and containerPort + # from 8080 to 8443. + # - --metrics-secure-serving + # - --metrics-bind-address=:8443 + # - --metrics-cert-dir=/etc/metrics-certs + # - --tls-min-version=VersionTLS12 + # - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + ports: + - name: metrics + containerPort: 8080 # Change to 8443 when --metrics-secure-serving is enabled. + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP + # Uncomment when --metrics-secure-serving is enabled: + # volumeMounts: + # - name: metrics-certs + # mountPath: /etc/metrics-certs + # readOnly: true + # volumes: + # - name: metrics-certs + # secret: + # secretName: agent-sandbox-metrics-tls + +--- +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: sandboxes.agents.x-k8s.io +spec: + group: agents.x-k8s.io + names: + kind: Sandbox + listKind: SandboxList + plural: sandboxes + shortNames: + - sandbox + singular: sandbox + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + failedPodPolicy: + default: Retain + enum: + - Retain + - Recreate + type: string + operatingMode: + default: Running + enum: + - Running + - Suspended + type: string + podTemplate: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + activeDeadlineSeconds: + format: int64 + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + type: boolean + containers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + bindMountOptions: + items: + type: string + type: array + x-kubernetes-list-type: set + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + properties: + nameservers: + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + type: string + enableServiceLinks: + type: boolean + ephemeralContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + targetContainerName: + type: string + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + bindMountOptions: + items: + type: string + type: array + x-kubernetes-list-type: set + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + evictionResponders: + items: + properties: + name: + type: string + priority: + format: int32 + type: integer + required: + - name + - priority + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + items: + properties: + hostnames: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + type: boolean + hostNetwork: + type: boolean + hostPID: + type: boolean + hostUsers: + type: boolean + hostname: + type: string + hostnameOverride: + type: string + imagePullSecrets: + items: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + items: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + mode: + type: string + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + type: string + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + bindMountOptions: + items: + type: string + type: array + x-kubernetes-list-type: set + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + x-kubernetes-map-type: atomic + os: + properties: + name: + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + preemptionPolicy: + type: string + priority: + format: int32 + type: integer + priorityClassName: + type: string + readinessGates: + items: + properties: + conditionType: + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + items: + properties: + name: + type: string + resourceClaimName: + type: string + resourceClaimTemplateName: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + runtimeClassName: + type: string + schedulerName: + type: string + schedulingGates: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + schedulingGroup: + properties: + podGroupName: + type: string + type: object + securityContext: + properties: + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxChangePolicy: + type: string + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + type: string + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + serviceAccountName: + type: string + setHostnameAsFQDN: + type: boolean + shareProcessNamespace: + type: boolean + subdomain: + type: string + terminationGracePeriodSeconds: + format: int64 + type: integer + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + default: ext4 + type: string + kind: + type: string + readOnly: + default: false + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + defaultUser: + format: int64 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + user: + format: int64 + type: integer + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + defaultUser: + format: int64 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + user: + format: int64 + type: integer + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + properties: + medium: + type: string + mode: + format: int32 + type: integer + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + image: + properties: + pullPolicy: + type: string + reference: + type: string + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + default: default + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + defaultUser: + format: int64 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + user: + format: int64 + type: integer + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + user: + format: int64 + type: integer + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + user: + format: int64 + type: integer + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podCertificate: + properties: + certificateChainPath: + type: string + credentialBundlePath: + type: string + keyPath: + type: string + keyType: + type: string + maxExpirationSeconds: + format: int32 + type: integer + signerName: + type: string + user: + format: int64 + type: integer + userAnnotations: + additionalProperties: + type: string + type: object + required: + - keyType + - signerName + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + user: + format: int64 + type: integer + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + user: + format: int64 + type: integer + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + default: /etc/ceph/keyring + type: string + monitors: + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + default: xfs + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + default: ThinProvisioned + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + defaultUser: + format: int64 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + user: + format: int64 + type: integer + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + default: "" + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + required: + - spec + type: object + service: + type: boolean + shutdownPolicy: + default: Retain + enum: + - Delete + - Retain + type: string + shutdownTime: + format: date-time + type: string + volumeClaimTemplates: + items: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + name: + maxLength: 63 + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: metadata.name must not be empty + rule: has(self.metadata) && has(self.metadata.name) && size(self.metadata.name) + > 0 + maxItems: 128 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: volumeClaimTemplates names must be unique + rule: self.all(a, !has(a.metadata) || !has(a.metadata.name) || self.exists_one(b, + has(b.metadata) && has(b.metadata.name) && b.metadata.name == + a.metadata.name)) + required: + - podTemplate + type: object + x-kubernetes-validations: + - message: volumeClaimTemplates is immutable + rule: has(self.volumeClaimTemplates) == has(oldSelf.volumeClaimTemplates) + && (!has(self.volumeClaimTemplates) || self.volumeClaimTemplates == + oldSelf.volumeClaimTemplates) + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + nodeName: + type: string + podIPs: + items: + type: string + type: array + selector: + type: string + service: + type: string + serviceFQDN: + type: string + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: metadata.name must not exceed 63 characters when spec.service is + true + rule: '!has(self.spec.service) || !self.spec.service || size(self.metadata.name) + <= 63' + served: true + storage: true + subresources: + status: {} +--- +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: agent-sandbox-controller +rules: +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - pods + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch +- apiGroups: + - agents.x-k8s.io + resources: + - sandboxes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - agents.x-k8s.io + resources: + - sandboxes/finalizers + - sandboxes/status + verbs: + - get + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - patch + - update + - watch +--- diff --git a/charts/agent-sandbox/source-lock.yaml b/charts/agent-sandbox/source-lock.yaml new file mode 100644 index 000000000..2a7486831 --- /dev/null +++ b/charts/agent-sandbox/source-lock.yaml @@ -0,0 +1,26 @@ +lockVersion: 1 +upstream: + repository: https://github.com/kubernetes-sigs/agent-sandbox + release: v1.0.9 + commit: b1277676f8bf0e82ea299928997ecf5d696a92f1 + manifestURL: https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v1.0.9/sandbox.yaml + manifestPath: files/sandbox.yaml + manifestSHA256: 3eeb27fa821659593ba92c5454571f08c84abb07096bb9d372c1c6c750f6d7d2 + licensePath: files/LICENSE +controller: + repository: registry.k8s.io/agent-sandbox/agent-sandbox-controller + tag: v1.0.9 + digest: sha256:ef1a1a3076afc996e711c44b388fbb7ffb7cd6858953d7e685166b299d77bd38 + mediaType: application/vnd.oci.image.index.v1+json +api: + group: agents.x-k8s.io + kind: Sandbox + servedVersions: [v1beta1] + storageVersion: v1beta1 +consumer: + repository: https://github.com/opendatahub-io/openshell + release: v0.1.2-rhaiv.6 + commit: 305c8b3d7bbdb97a6f2f7264c3ad43b0087c307f + reviewedFields: [podTemplate, volumeClaimTemplates, operatingMode] + qualification: source-reviewed-only +verifiedOn: "2026-11-04" diff --git a/charts/agent-sandbox/templates/manifest.yaml b/charts/agent-sandbox/templates/manifest.yaml new file mode 100644 index 000000000..0bff753d7 --- /dev/null +++ b/charts/agent-sandbox/templates/manifest.yaml @@ -0,0 +1,45 @@ +{{- if .Values.enabled }} +{{- if ne .Release.Namespace "agent-sandbox-system" }} +{{- fail "Agent Sandbox is a cluster-wide dependency: install once in namespace agent-sandbox-system" }} +{{- end }} +{{- $lock := .Files.Get "source-lock.yaml" | fromYaml }} +{{- $manifest := .Files.Get $lock.upstream.manifestPath }} +{{- if ne (sha256sum $manifest) $lock.upstream.manifestSHA256 }} +{{- fail "vendored Agent Sandbox manifest differs from source-lock.yaml" }} +{{- end }} +{{- range $part := splitList "\n---" $manifest }} +{{- $object := fromYaml $part }} +{{- if and $object.kind (ne $object.kind "Namespace") }} +{{- $meta := $object.metadata }} +{{- $labels := default dict $meta.labels }} +{{- $_ := set $labels "app.kubernetes.io/name" "agent-sandbox" }} +{{- $_ := set $labels "app.kubernetes.io/version" $.Chart.AppVersion }} +{{- $_ := set $meta "labels" $labels }} +{{- $annotations := default dict $meta.annotations }} +{{- $wave := "-1" }} +{{- if eq $object.kind "CustomResourceDefinition" }} +{{- $wave = "-2" }} +{{- else if eq $object.kind "Deployment" }} +{{- $wave = "0" }} +{{- end }} +{{- $_ := set $annotations "argocd.argoproj.io/sync-wave" $wave }} +{{- if eq $object.kind "CustomResourceDefinition" }} +{{- $_ := set $annotations "helm.sh/resource-policy" "keep" }} +{{- $_ := set $annotations "argocd.argoproj.io/sync-options" "Prune=false,Delete=false" }} +{{- end }} +{{- $_ := set $meta "annotations" $annotations }} +{{- if eq $object.kind "Deployment" }} +{{- $pod := $object.spec.template.spec }} +{{- $container := index $pod.containers 0 }} +{{- $_ := set $container "image" (printf "%s@%s" $.Values.controller.imageRepository $lock.controller.digest) }} +{{- $_ := set $container "resources" $.Values.controller.resources }} +{{- $_ := set $pod "securityContext" (dict "runAsNonRoot" true "seccompProfile" (dict "type" "RuntimeDefault")) }} +{{- $_ := set $container "securityContext" (dict "allowPrivilegeEscalation" false "capabilities" (dict "drop" (list "ALL"))) }} +{{- $_ := set $container "readinessProbe" (dict "httpGet" (dict "path" "/readyz" "port" "healthz") "initialDelaySeconds" 5 "periodSeconds" 10) }} +{{- $_ := set $container "livenessProbe" (dict "httpGet" (dict "path" "/healthz" "port" "healthz") "initialDelaySeconds" 15 "periodSeconds" 20) }} +{{- end }} +--- +{{ toYaml $object }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/agent-sandbox/values.schema.json b/charts/agent-sandbox/values.schema.json new file mode 100644 index 000000000..84dbfc9d9 --- /dev/null +++ b/charts/agent-sandbox/values.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "controller": { + "type": "object", + "additionalProperties": false, + "properties": { + "imageRepository": { + "type": "string", + "pattern": "^[a-zA-Z0-9.-]+(:[0-9]+)?/[a-zA-Z0-9_./-]+$" + }, + "resources": {"type": "object"} + }, + "required": ["imageRepository", "resources"] + } + }, + "required": ["enabled", "controller"] +} diff --git a/charts/agent-sandbox/values.yaml b/charts/agent-sandbox/values.yaml new file mode 100644 index 000000000..df8f9d972 --- /dev/null +++ b/charts/agent-sandbox/values.yaml @@ -0,0 +1,13 @@ +# Install once per cluster, only when no other platform owner provides Agent Sandbox. +# This dependency alone does not qualify SAW's container runtime. +enabled: false + +controller: + # A mirror may change the repository, but the digest remains source-locked. + imageRepository: registry.k8s.io/agent-sandbox/agent-sandbox-controller + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 512Mi diff --git a/charts/governance-interceptor/templates/networkpolicy.yaml b/charts/governance-interceptor/templates/networkpolicy.yaml index 39c8b128d..3d96881e6 100644 --- a/charts/governance-interceptor/templates/networkpolicy.yaml +++ b/charts/governance-interceptor/templates/networkpolicy.yaml @@ -29,4 +29,16 @@ spec: podSelector: matchLabels: kubevirt.io: virt-launcher + {{- range .Values.kubernetesGateways }} + {{- if or (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" (.namespace | default ""))) (gt (len .namespace) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" (.gatewayId | default ""))) (gt (len .gatewayId) 19) }} + {{- fail "kubernetesGateways requires exact namespace and gatewayId DNS labels" }} + {{- end }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .namespace | quote }} + podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .gatewayId | quote }} + openshell.pattern/role: gateway + {{- end }} {{- end }} diff --git a/charts/governance-interceptor/values.yaml b/charts/governance-interceptor/values.yaml index 19a0352e4..5b111ade4 100644 --- a/charts/governance-interceptor/values.yaml +++ b/charts/governance-interceptor/values.yaml @@ -47,6 +47,11 @@ sawNamespaceLabel: key: openshell.pattern/saw value: "true" +# Explicit administrator-approved Kubernetes gateways. Empty preserves VM-only +# ingress. Each entry requires an exact namespace and Helm release/gateway ID. +# This enables network reachability, not policy or driver-config authorization. +kubernetesGateways: [] + # --- engine: apf ------------------------------------------------------------- apf: # The APF chart (private OCI chart on GHCR). diff --git a/charts/openshell-keycloak/files/owner-resolver.py b/charts/openshell-keycloak/files/owner-resolver.py index 6c7e7aa01..16a627448 100644 --- a/charts/openshell-keycloak/files/owner-resolver.py +++ b/charts/openshell-keycloak/files/owner-resolver.py @@ -16,6 +16,7 @@ import urllib.request import saw_tls +from saw_keycloak_admin import ensure_admin_api_scope SA = "/var/run/secrets/kubernetes.io/serviceaccount" USER_RE = re.compile(r"[a-z0-9]([-a-z0-9]{0,17}[a-z0-9])?") @@ -128,14 +129,9 @@ def bootstrap(): if not roles: role = kc_call(url, realm, token, f"/clients/{management_id}/roles/view-users", ctx=ctx) kc_call(url, realm, token, path, "POST", [role], ctx) - # Validate role grants before enabling Admin API tokens. - admin_api_scopes = kc_call(url, realm, token, "/client-scopes", ctx=ctx) or [] - admin_api_scope = next((s for s in admin_api_scopes if s.get("name") == "kc-admin-api"), None) - if admin_api_scope is not None: - default_scopes = kc_call(url, realm, token, f"/clients/{client['id']}/default-client-scopes", ctx=ctx) or [] - if not any(s.get("name") == "kc-admin-api" for s in default_scopes): - kc_call(url, realm, token, - f"/clients/{client['id']}/default-client-scopes/{admin_api_scope['id']}", "PUT", ctx=ctx) + ensure_admin_api_scope( + lambda method, path, body=None: kc_call(url, realm, token, path, method, body, ctx), + client["id"]) secret = kc_call(url, realm, token, f"/clients/{client['id']}/client-secret", ctx=ctx)["value"] fd = os.open(os.environ["OUT"], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "w", encoding="utf-8") as stream: diff --git a/charts/openshell-keycloak/files/redirect-registrar.py b/charts/openshell-keycloak/files/redirect-registrar.py index 5f913e025..fd4539359 100644 --- a/charts/openshell-keycloak/files/redirect-registrar.py +++ b/charts/openshell-keycloak/files/redirect-registrar.py @@ -36,6 +36,7 @@ import urllib.request import saw_tls +from saw_keycloak_admin import ensure_admin_api_scope SA = "/var/run/secrets/kubernetes.io/serviceaccount" # The URIs and origins this registrar set, so it removes only its own. @@ -167,13 +168,7 @@ def bootstrap(): if not any(r.get("name") == "manage-clients" for r in have): kc("POST", f"/users/{account['id']}/role-mappings/clients/{management['id']}", [role]) log(f"{client_id} may manage the clients of realm {realm}") - # Updated Keycloak versions require this scope as well as the role. - scopes = kc("GET", "/client-scopes") or [] - admin_scope = next((s for s in scopes if s.get("name") == "kc-admin-api"), None) - if admin_scope is not None: - defaults = kc("GET", f"/clients/{client['id']}/default-client-scopes") or [] - if not any(s.get("name") == "kc-admin-api" for s in defaults): - kc("PUT", f"/clients/{client['id']}/default-client-scopes/{admin_scope['id']}") + ensure_admin_api_scope(kc, client["id"]) secret = kc("GET", f"/clients/{client['id']}/client-secret")["value"] out = env("OUT") fd = os.open(out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) diff --git a/charts/openshell-keycloak/files/saw_keycloak_admin.py b/charts/openshell-keycloak/files/saw_keycloak_admin.py new file mode 100644 index 000000000..90c1f4bd1 --- /dev/null +++ b/charts/openshell-keycloak/files/saw_keycloak_admin.py @@ -0,0 +1,36 @@ +"""Shared bootstrap of the built-in Keycloak Admin REST API audience scope. + +Canonical source here; ``make catalog`` packages the same module for RHDH. +The caller supplies its existing administrator-authenticated API transport. +This helper neither requests credentials nor grants administrative roles. +""" +from urllib.parse import quote + +ADMIN_API_SCOPE = "kc-admin-api" + + +def ensure_admin_api_scope(call, client_uuid): + """Assign the existing built-in scope without replacing other defaults. + + Keycloak 26.6.8 requires both this scope and its mapped Admin API audience. + Older servers do not have the scope: leave their configuration unchanged. + Lookup/authorization failures propagate; they are never treated as absence. + ``call`` accepts (method, realm-relative path, optional JSON body). + """ + scopes = call("GET", "/client-scopes") + if not isinstance(scopes, list): + raise RuntimeError("invalid Keycloak client scope response") + matching = [scope for scope in scopes if scope.get("name") == ADMIN_API_SCOPE] + if not matching: + return False + if (len(matching) != 1 or matching[0].get("protocol") != "openid-connect" + or not isinstance(matching[0].get("id"), str) or not matching[0]["id"]): + raise RuntimeError("invalid Keycloak Admin API scope") + scope_id = matching[0]["id"] + path = "/clients/" + quote(client_uuid, safe="") + "/default-client-scopes" + assigned = call("GET", path) + if not isinstance(assigned, list): + raise RuntimeError("invalid Keycloak default client scope response") + if not any(scope.get("id") == scope_id for scope in assigned): + call("PUT", path + "/" + quote(scope_id, safe="")) + return True diff --git a/charts/openshell-keycloak/templates/owner-resolver.yaml b/charts/openshell-keycloak/templates/owner-resolver.yaml index 4c52490d2..1495f7843 100644 --- a/charts/openshell-keycloak/templates/owner-resolver.yaml +++ b/charts/openshell-keycloak/templates/owner-resolver.yaml @@ -92,6 +92,8 @@ kind: ConfigMap metadata: name: {{ $name }} data: + saw_keycloak_admin.py: | +{{ .Files.Get "files/saw_keycloak_admin.py" | indent 4 }} ca-bundle.crt: | {{ $r.caBundle | default "" | indent 4 }} saw_tls.py: | @@ -115,7 +117,7 @@ spec: labels: app.kubernetes.io/name: {{ $name }} annotations: - checksum/script: {{ print (.Files.Get "files/owner-resolver.py") (.Files.Get "files/saw_tls.py") ($r.caBundle | default "") | sha256sum }} + checksum/script: {{ print (.Files.Get "files/owner-resolver.py") (.Files.Get "files/saw_tls.py") (.Files.Get "files/saw_keycloak_admin.py") ($r.caBundle | default "") | sha256sum }} spec: serviceAccountName: {{ $name }} securityContext: diff --git a/charts/openshell-keycloak/templates/redirect-registrar.yaml b/charts/openshell-keycloak/templates/redirect-registrar.yaml index ec382cf48..6f2759cd4 100644 --- a/charts/openshell-keycloak/templates/redirect-registrar.yaml +++ b/charts/openshell-keycloak/templates/redirect-registrar.yaml @@ -87,6 +87,8 @@ metadata: labels: app.kubernetes.io/name: {{ $name }} data: + saw_keycloak_admin.py: | +{{ .Files.Get "files/saw_keycloak_admin.py" | indent 4 }} ca-bundle.crt: | {{ $r.caBundle | default "" | indent 4 }} saw_tls.py: | @@ -112,7 +114,7 @@ spec: labels: app.kubernetes.io/name: {{ $name }} annotations: - checksum/script: {{ print (.Files.Get "files/redirect-registrar.py") (.Files.Get "files/saw_tls.py") ($r.caBundle | default "") | sha256sum }} + checksum/script: {{ print (.Files.Get "files/redirect-registrar.py") (.Files.Get "files/saw_tls.py") (.Files.Get "files/saw_keycloak_admin.py") ($r.caBundle | default "") | sha256sum }} spec: serviceAccountName: {{ $name }} securityContext: diff --git a/charts/openshell-rhdh/files/create-workspace-for-user.yaml b/charts/openshell-rhdh/files/create-workspace-for-user.yaml index 108492cd4..a1d46dcdb 100644 --- a/charts/openshell-rhdh/files/create-workspace-for-user.yaml +++ b/charts/openshell-rhdh/files/create-workspace-for-user.yaml @@ -21,8 +21,16 @@ spec: Administrators only; your own name for your workspace. The user signs in with their own password.} - title: Workspace profile description: The workspace is named after the user (namespace saw-). - required: [profile] + required: [profile, deploymentMode] properties: + deploymentMode: + title: Deployment + type: string + enum: [vm] + enumNames: [Virtual machine] + default: vm + description: Available deployment options are set by your administrator. An existing workspace + keeps its deployment type. profile: title: Profile type: string @@ -52,9 +60,16 @@ spec: inference__url: {title: 'inference: Endpoint base URL (OpenAI-compatible, ending in /v1)', type: string, pattern: '^https?://'} inference__model: {title: 'inference: Model', type: string} - web_search__api_key: {title: 'web-search: API key', type: string, 'ui:field': Secret, description: Stored - in Vault for your workspace only (provider brave)} - required: [inference__api_key, inference__url, inference__model, web_search__api_key] + required: [inference__api_key, inference__url, inference__model] + allOf: + - if: + properties: + deploymentMode: {const: vm} + then: + properties: + web_search__api_key: {title: 'web-search: API key', type: string, 'ui:field': Secret, + description: Stored in Vault for your workspace only (provider brave)} + required: [web_search__api_key] - properties: profile: {const: data-science} uiRoutes: {title: Sandbox web UIs, type: 'null', description: 'Published on their own route, @@ -99,12 +114,13 @@ spec: labels: {saw.redhat.com/request: 'true'} type: Opaque stringData: {action: create, token: '${{ secrets.backstageToken }}', profile: '${{ parameters.profile - }}', forUser: '${{ parameters.forUser }}', inference.api_key: '${{ secrets.inference__api_key - }}', inference.url: '${{ parameters.inference__url }}', inference.model: '${{ parameters.inference__model - }}', web-search.api_key: '${{ secrets.web_search__api_key }}', inference.cuda_model: '${{ - parameters.inference__cuda_model }}', gmail.client_id: '${{ secrets.gmail__client_id }}', - gmail.client_secret: '${{ secrets.gmail__client_secret }}', gmail.refresh_token: '${{ secrets.gmail__refresh_token - }}', slack.bot_token: '${{ secrets.slack__bot_token }}'} + }}', deploymentMode: '${{ parameters.deploymentMode }}', forUser: '${{ parameters.forUser + }}', inference.api_key: '${{ secrets.inference__api_key }}', inference.url: '${{ parameters.inference__url + }}', inference.model: '${{ parameters.inference__model }}', web-search.api_key: '${{ secrets.web_search__api_key + }}', inference.cuda_model: '${{ parameters.inference__cuda_model }}', gmail.client_id: '${{ + secrets.gmail__client_id }}', gmail.client_secret: '${{ secrets.gmail__client_secret }}', + gmail.refresh_token: '${{ secrets.gmail__refresh_token }}', slack.bot_token: '${{ secrets.slack__bot_token + }}'} - id: run name: Start the pipeline action: http:backstage:request @@ -147,7 +163,7 @@ spec: each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] - id: task-vm - name: Argo CD creates the VM + name: Provision the workspace runtime action: http:backstage:request input: method: GET @@ -158,7 +174,7 @@ spec: each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] - id: task-vm-running - name: Start the VM + name: Start the workspace runtime action: http:backstage:request input: method: GET @@ -170,7 +186,7 @@ spec: 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51] - id: task-sandboxes - name: Install OpenShell and the sandboxes (about 10 minutes) + name: Wait for OpenShell sandboxes and workspace access action: http:backstage:request input: method: GET diff --git a/charts/openshell-rhdh/files/create-workspace.yaml b/charts/openshell-rhdh/files/create-workspace.yaml index 4cf0de221..64e754a7f 100644 --- a/charts/openshell-rhdh/files/create-workspace.yaml +++ b/charts/openshell-rhdh/files/create-workspace.yaml @@ -4,9 +4,8 @@ kind: Template metadata: name: create-saw-workspace title: Create or update my agent workspace - description: 'Your own Secure Agent Workspace: a VM with OpenShell and the OpenClaw / NemoClaw sandboxes - of a SAW-BOM profile. If you have one, this changes its profile or keys. Keys go to Vault, not to - Git.' + description: Your own Secure Agent Workspace with OpenShell and the OpenClaw / NemoClaw sandboxes of + a SAW-BOM profile. If you have one, this changes its profile or keys. Keys go to Vault, not to Git. tags: [openshell, openclaw, nemoclaw, secure-agent-workspace] spec: owner: user:default/admin @@ -14,8 +13,16 @@ spec: parameters: - title: Workspace profile description: The workspace is named after you (namespace saw-). - required: [profile] + required: [profile, deploymentMode] properties: + deploymentMode: + title: Deployment + type: string + enum: [vm] + enumNames: [Virtual machine] + default: vm + description: Available deployment options are set by your administrator. An existing workspace + keeps its deployment type. profile: title: Profile type: string @@ -45,9 +52,16 @@ spec: inference__url: {title: 'inference: Endpoint base URL (OpenAI-compatible, ending in /v1)', type: string, pattern: '^https?://'} inference__model: {title: 'inference: Model', type: string} - web_search__api_key: {title: 'web-search: API key', type: string, 'ui:field': Secret, description: Stored - in Vault for your workspace only (provider brave)} - required: [inference__api_key, inference__url, inference__model, web_search__api_key] + required: [inference__api_key, inference__url, inference__model] + allOf: + - if: + properties: + deploymentMode: {const: vm} + then: + properties: + web_search__api_key: {title: 'web-search: API key', type: string, 'ui:field': Secret, + description: Stored in Vault for your workspace only (provider brave)} + required: [web_search__api_key] - properties: profile: {const: data-science} uiRoutes: {title: Sandbox web UIs, type: 'null', description: 'Published on their own route, @@ -92,12 +106,12 @@ spec: labels: {saw.redhat.com/request: 'true'} type: Opaque stringData: {action: create, token: '${{ secrets.backstageToken }}', profile: '${{ parameters.profile - }}', inference.api_key: '${{ secrets.inference__api_key }}', inference.url: '${{ parameters.inference__url - }}', inference.model: '${{ parameters.inference__model }}', web-search.api_key: '${{ secrets.web_search__api_key - }}', inference.cuda_model: '${{ parameters.inference__cuda_model }}', gmail.client_id: '${{ - secrets.gmail__client_id }}', gmail.client_secret: '${{ secrets.gmail__client_secret }}', - gmail.refresh_token: '${{ secrets.gmail__refresh_token }}', slack.bot_token: '${{ secrets.slack__bot_token - }}'} + }}', deploymentMode: '${{ parameters.deploymentMode }}', inference.api_key: '${{ secrets.inference__api_key + }}', inference.url: '${{ parameters.inference__url }}', inference.model: '${{ parameters.inference__model + }}', web-search.api_key: '${{ secrets.web_search__api_key }}', inference.cuda_model: '${{ + parameters.inference__cuda_model }}', gmail.client_id: '${{ secrets.gmail__client_id }}', + gmail.client_secret: '${{ secrets.gmail__client_secret }}', gmail.refresh_token: '${{ secrets.gmail__refresh_token + }}', slack.bot_token: '${{ secrets.slack__bot_token }}'} - id: run name: Start the pipeline action: http:backstage:request @@ -140,7 +154,7 @@ spec: each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] - id: task-vm - name: Argo CD creates the VM + name: Provision the workspace runtime action: http:backstage:request input: method: GET @@ -151,7 +165,7 @@ spec: each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] - id: task-vm-running - name: Start the VM + name: Start the workspace runtime action: http:backstage:request input: method: GET @@ -163,7 +177,7 @@ spec: 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51] - id: task-sandboxes - name: Install OpenShell and the sandboxes (about 10 minutes) + name: Wait for OpenShell sandboxes and workspace access action: http:backstage:request input: method: GET diff --git a/charts/openshell-rhdh/files/delete-workspace-for-user.yaml b/charts/openshell-rhdh/files/delete-workspace-for-user.yaml index 360563a68..008aa1b8f 100644 --- a/charts/openshell-rhdh/files/delete-workspace-for-user.yaml +++ b/charts/openshell-rhdh/files/delete-workspace-for-user.yaml @@ -4,8 +4,8 @@ kind: Template metadata: name: delete-saw-workspace-for-user title: Delete a user's agent workspace - description: 'Administrators: removes a user''s Secure Agent Workspace, its VM, its sandboxes and everything - in them, and the user''s keys in Vault. This cannot be undone.' + description: 'Administrators: removes a user''s Secure Agent Workspace, its runtime, its sandboxes and + the user''s keys in Vault. Retained audit history is separate. This cannot be undone.' tags: [openshell, secure-agent-workspace, admin] labels: {saw.redhat.com/admin: 'true'} spec: @@ -25,8 +25,8 @@ spec: catalogFilter: {kind: Component, spec.type: agent-workspace} defaultKind: Component allowArbitraryValues: false - confirm: {title: 'I understand that this workspace, its VM and its data are deleted', type: boolean, - const: true} + confirm: {title: 'I understand that the workspace runtime, sandbox data and workspace keys will + be deleted', type: boolean, const: true} steps: - id: request name: Submit the request @@ -75,7 +75,7 @@ spec: continueOnBadResponse: true each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - id: task-argo-cd-removes - name: Argo CD removes the namespace and the VM + name: Wait for Argo CD to remove the workspace runtime action: http:backstage:request input: method: GET @@ -119,5 +119,5 @@ spec: output: text: - {title: Workspace deleted, content: 'Pipeline run **${{ steps.run.output.body.metadata.name }}** - deleted workspace ${{ parameters.workspace | parseEntityRef | pick(''name'') }}: its VM, namespace - and keys are gone.'} + deleted workspace ${{ parameters.workspace | parseEntityRef | pick(''name'') }}: its workspace + compute and access have been removed.'} diff --git a/charts/openshell-rhdh/files/delete-workspace.yaml b/charts/openshell-rhdh/files/delete-workspace.yaml index 1308027e8..6118a5fa7 100644 --- a/charts/openshell-rhdh/files/delete-workspace.yaml +++ b/charts/openshell-rhdh/files/delete-workspace.yaml @@ -4,8 +4,8 @@ kind: Template metadata: name: delete-saw-workspace title: Delete my agent workspace - description: 'Removes your Secure Agent Workspace: its VM, its sandboxes and everything in them, and - your keys in Vault. This cannot be undone.' + description: 'Removes your Secure Agent Workspace: its runtime, its sandboxes and the data in them, + and your keys in Vault. Retained audit history is separate. This cannot be undone.' tags: [openshell, secure-agent-workspace] labels: {saw.redhat.com/owners: 'true'} spec: @@ -25,8 +25,8 @@ spec: catalogFilter: {kind: Component, spec.type: agent-workspace} defaultKind: Component allowArbitraryValues: false - confirm: {title: 'I understand that this workspace, its VM and its data are deleted', type: boolean, - const: true} + confirm: {title: 'I understand that the workspace runtime, sandbox data and workspace keys will + be deleted', type: boolean, const: true} steps: - id: request name: Submit the request @@ -75,7 +75,7 @@ spec: continueOnBadResponse: true each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - id: task-argo-cd-removes - name: Argo CD removes the namespace and the VM + name: Wait for Argo CD to remove the workspace runtime action: http:backstage:request input: method: GET @@ -119,5 +119,5 @@ spec: output: text: - {title: Workspace deleted, content: 'Pipeline run **${{ steps.run.output.body.metadata.name }}** - deleted workspace ${{ parameters.workspace | parseEntityRef | pick(''name'') }}: its VM, namespace - and keys are gone.'} + deleted workspace ${{ parameters.workspace | parseEntityRef | pick(''name'') }}: its workspace + compute and access have been removed.'} diff --git a/charts/openshell-rhdh/files/keycloak-client.py b/charts/openshell-rhdh/files/keycloak-client.py index 70e0c6246..387482ca2 100644 --- a/charts/openshell-rhdh/files/keycloak-client.py +++ b/charts/openshell-rhdh/files/keycloak-client.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Create or update RHDH's confidential OIDC client in the openshell realm. -Runs as an Argo CD PostSync hook, so it also fixes a realm that existed +Runs as an Argo CD Sync hook, so it also fixes a realm that existed before the portal (a realm import only runs once). The client secret comes from the rhdh-oidc Secret (from Vault); the Keycloak admin credentials from the operator's -initial-admin Secret, read through the Kubernetes @@ -26,6 +26,8 @@ import urllib.parse import urllib.request +from saw_keycloak_admin import ensure_admin_api_scope + SA = "/var/run/secrets/kubernetes.io/serviceaccount" @@ -108,15 +110,10 @@ def grant_catalog_reader(admin, client_uuid, token, ctx): roles = [call(f"{admin}/clients/{rm['id']}/roles/{name}", token=token, ctx=ctx) for name in READER_ROLES] call(f"{admin}/users/{sa['id']}/role-mappings/clients/{rm['id']}", method="POST", body=roles, token=token, ctx=ctx) - # Updated Keycloak versions require this scope as well as the reader roles. - scopes = call(f"{admin}/client-scopes", token=token, ctx=ctx) or [] - admin_scope = next((s for s in scopes if s.get("name") == "kc-admin-api"), None) - if admin_scope is not None: - defaults = call(f"{admin}/clients/{client_uuid}/default-client-scopes", token=token, ctx=ctx) or [] - if not any(s.get("name") == "kc-admin-api" for s in defaults): - call(f"{admin}/clients/{client_uuid}/default-client-scopes/{admin_scope['id']}", - method="PUT", token=token, ctx=ctx) - print(f"Service account {sa.get('username')} may read users and groups ({', '.join(READER_ROLES)})") + ensure_admin_api_scope( + lambda method, path, body=None: call(admin + path, method=method, body=body, token=token, ctx=ctx), + client_uuid) + def everyone_in_group(admin, name, token, ctx): diff --git a/charts/openshell-rhdh/files/portal.py b/charts/openshell-rhdh/files/portal.py index c26f6e1ec..5e8c5232a 100644 --- a/charts/openshell-rhdh/files/portal.py +++ b/charts/openshell-rhdh/files/portal.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Self-service workspaces for the Secure Agent Workspace pattern. -Standard library only; one file: +Standard library only; packaged with the shared workspace projector: portal.py create REQUEST pipeline saw-workspace-create, task register portal.py wait- USER [S] its later tasks: wait for apps, vm, running, @@ -42,6 +42,8 @@ delete the request Secret. """ import base64 +from concurrent.futures import ThreadPoolExecutor +import threading import calendar import hashlib import http.server @@ -82,6 +84,53 @@ class PortalError(Exception): """A request that must be refused; the message is shown in the PipelineRun.""" +# Packaged from charts/saw-users/files/workspace.py by make catalog. +import importlib.util as _importlib +from pathlib import Path as _Path +_workspace_spec = _importlib.spec_from_file_location("saw_workspace", _Path(__file__).with_name("workspace.py")) +_workspace = _importlib.module_from_spec(_workspace_spec) +_workspace_spec.loader.exec_module(_workspace) + + +_preflight_spec = _importlib.spec_from_file_location("saw_preflight", _Path(__file__).with_name("preflight.py")) +_preflight = _importlib.module_from_spec(_preflight_spec) +_preflight_spec.loader.exec_module(_preflight) + + +def _project(function, *args, **kwargs): + try: + return function(*args, **kwargs) + except _workspace.WorkspaceError as exc: + raise PortalError(str(exc)) from exc + + +def enabled_catalog_sandboxes(profile): + return _workspace.enabled_catalog_sandboxes(profile) + + +def sandbox_targets(user, registration, profile, *, model_lifecycle=False): + return _workspace.sandbox_targets(user, registration, profile, model_lifecycle=model_lifecycle) + + +def public_model_inputs(profile, secrets): + return _project(_workspace.public_model_inputs, profile, secrets) + + +def approved_endpoint(profile, secrets, boundary): + return _project(_workspace.approved_endpoint, profile, secrets, boundary) + + +def enrolled_site(site, workspace, read): + def observed(path): + try: + return read(path) + except HttpError as exc: + if exc.code == 403: # first sync, before the named reader grant exists + return {} + raise + return _project(_workspace.enrolled_site, site, workspace, observed if read else None) + + class NotARequest(PortalError): """The named Secret is not a request: it is not deleted.""" @@ -171,6 +220,11 @@ def verify_jwt(token, jwks, what, now=None, leeway=0): header, payload, signed, signature = decode_jwt(token) if header.get("alg") != "ES256": raise PortalError(f"{what}: unsupported algorithm {header.get('alg')!r} (expected ES256)") + kid = header.get("kid") + if not isinstance(kid, str) or not kid: + raise PortalError(f"{what}: missing signing key ID") + if callable(jwks): + jwks = jwks(kid) keys = [k for k in jwks.get("keys", []) if k.get("kid") == header.get("kid") and k.get("kty") == "EC" and k.get("crv") == "P-256"] if not keys: @@ -265,8 +319,31 @@ def kube(): return Http(f"https://{host}:{port}", token, cafile=f"{SA_DIR}/ca.crt") -def fetch_json(url, insecure=False): - return Http(url, insecure=insecure).call("GET", "") +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args): + return None + + +def fetch_json(url): + """Read RHDH signing keys over verified TLS without following redirects.""" + endpoint = urllib.parse.urlsplit(url) + if (endpoint.scheme != "https" or not endpoint.hostname or endpoint.username is not None + or endpoint.password is not None or endpoint.query or endpoint.fragment): + raise PortalError("RHDH signing keys require a configured HTTPS endpoint") + opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler( + context=ssl.create_default_context())) + try: + with opener.open(urllib.request.Request(url, headers={"Accept": "application/json"}), timeout=15) as response: + raw = response.read(1024 * 1024 + 1) + if len(raw) > 1024 * 1024: + raise PortalError("RHDH signing key response is too large") + data = json.loads(raw) + except (OSError, ValueError) as exc: + raise PortalError("could not fetch RHDH signing keys from the configured HTTPS endpoint") from exc + if not isinstance(data, dict) or not isinstance(data.get("keys"), list) or not all( + isinstance(key, dict) for key in data["keys"]): + raise PortalError("invalid RHDH signing key response") + return data class Vault: @@ -309,6 +386,99 @@ def check_user(user): return user +# Request policy is operator-owned, never taken from a form or registry entry. +KUBERNETES_MODES = ("container", "kata") +DEPLOYMENT_MODES = ("vm", *KUBERNETES_MODES) +# A renderable foundation is not a qualified provisioning/readiness adapter. +# Expand only when runtime lifecycle, BOM and readiness are implemented. +PROVISIONABLE_MODES = ("vm",) + + +def container_site(user): + """Trusted site inventory, never names/URLs supplied by the request.""" + sites = json.loads(os.environ.get("SAW_CONTAINER_SITES", "{}")) + site = sites.get(user) + if not isinstance(site, dict) or not re.fullmatch( + r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", str(site.get("subject", ""))): + return None + return site + + +def container_profiles(site): + automatic = site.get("automaticEnrollment", {}) + return automatic.get("approvedProfiles", {}) + + +def registered_links(entry, catalog, domain): + if entry_deployment_mode(entry) in KUBERNETES_MODES: + if entry.get("desiredState") == "Stopped": + return [] + return sandbox_ui_links(entry["name"], entry.get("profiles", []), catalog, domain) if domain else [] + return ui_links(entry["name"], entry.get("profiles", []), catalog, domain) + + +def container_status(k8s, user, entry, argo_ns, domain, ping, catalog=None): + reg = entry_generation(user, entry) + normalized = {**entry, "name": user, "registration": reg, + "installationID": installation_id(user, reg) if reg else ""} + def read(path): + try: + return k8s.call("GET", path, ok404=True) + except HttpError as exc: + if exc.code == 403: + return {} + raise + return _workspace.container_status(read, normalized, container_site(user), argo_ns, domain, ping, catalog) + + +def installation_id(user, registration): + import uuid + if not GENERATION_RE.fullmatch(registration): + raise PortalError("container requires a generation-scoped registration") + return str(uuid.uuid5(uuid.NAMESPACE_URL, "urn:saw:portal:" + check_user(user) + ":" + registration)) + + +def deployment_modes(): + try: + modes = json.loads(os.environ.get("SAW_DEPLOYMENT_MODES", '["vm"]')) + except ValueError: + raise PortalError("SAW_DEPLOYMENT_MODES must be a JSON list") from None + if (not isinstance(modes, list) or not modes or "vm" not in modes + or any(not isinstance(m, str) or m not in DEPLOYMENT_MODES for m in modes) + or len(set(modes)) != len(modes)): + raise PortalError("SAW_DEPLOYMENT_MODES must contain vm and unique supported modes: vm, container, kata") + return tuple(modes) + + +def validate_deployment_mode(mode): + if not isinstance(mode, str) or mode not in DEPLOYMENT_MODES: + raise PortalError("deploymentMode must be one of vm, container, kata") + return mode + + +def request_deployment_mode(data): + mode = validate_deployment_mode(data.get("deploymentMode", "vm")) + if mode not in deployment_modes(): + raise PortalError(f"deploymentMode={mode} is not enabled by the administrator") + return mode + + +def entry_deployment_mode(entry): + """Legacy registry entries are VMs, independently of later chart defaults.""" + values = entry.get("values", {}) + if not isinstance(values, dict): + raise PortalError("registry values must be an object") + deployment = values.get("deployment", {}) + if not isinstance(deployment, dict) or set(deployment) - {"mode"}: + raise PortalError("registry deployment must be an object with only mode") + return validate_deployment_mode(deployment.get("mode", "vm")) + + +def assert_mode_unchanged(entry, mode): + if entry_deployment_mode(entry) != mode: + raise PortalError("workspace deployment mode is immutable; an explicit migration is required") + + def parse_request(data, catalog): """The chosen profile and, for each Secret it reads, the field values. @@ -319,7 +489,10 @@ def parse_request(data, catalog): if profile not in catalog: raise PortalError(f"unknown profile {profile!r}; available: {', '.join(sorted(catalog))}") secrets, missing = {}, [] + allowed = catalog[profile].get("dedicatedEndpointSecrets") if request_deployment_mode(data) in KUBERNETES_MODES else None for name, spec in sorted(catalog[profile].get("secrets", {}).items()): + if allowed is not None and name not in allowed: + continue values = {} for field in spec.get("fields", []): value = (data.get(f"{name}.{field['key']}") or "").strip() @@ -373,13 +546,15 @@ def token_user(token, jwks=None, leeway=0): published keys. `jwks(path)` fetches a JWKS (the generator caches).""" if jwks is None: rhdh = env("RHDH_INTERNAL_URL").rstrip("/") - insecure = os.environ.get("RHDH_SKIP_VERIFY") == "true" - jwks = lambda path: fetch_json(rhdh + path, insecure=insecure) # noqa: E731 - scaffolder_jwks = None - if decode_jwt(token)[0].get("typ") == PLUGIN_TYP: - scaffolder_jwks = jwks("/api/scaffolder/.backstage/auth/v1/jwks.json") - return check_user(verify_backstage_token(token, jwks("/api/auth/.well-known/jwks.json"), - scaffolder_jwks, leeway=leeway)) + jwks = lambda path: fetch_json(rhdh + path) # noqa: E731 + def keys_for(path): + if isinstance(jwks, JwksCache): + return lambda kid: jwks(path, kid) + return lambda kid: jwks(path) + + return check_user(verify_backstage_token(token, keys_for("/api/auth/.well-known/jwks.json"), + keys_for("/api/scaffolder/.backstage/auth/v1/jwks.json"), + leeway=leeway)) def requester(data): @@ -432,14 +607,16 @@ def read_entry(cm): return entry if isinstance(entry, dict) else {} -def registry_entry(user, profile, generation=""): +def registry_entry(user, profile, generation="", mode="vm"): """The saw-users list entry for a portal workspace.""" + mode = validate_deployment_mode(mode) return {"name": user, "profiles": [profile], "ownerSubject": "", + "values": {"deployment": {"mode": mode}}, "vaultPrefix": f"{env('VAULT_KV_MOUNT', 'secret')}/data/{vault_prefix(user, generation)}", "pruneOnRemove": os.environ.get("PRUNE_ON_REMOVE", "true") == "true"} -def put_configmap(k8s, ns, name, data, labels=None, resource_version=None): +def put_configmap(k8s, ns, name, data, labels=None, resource_version=None, deployment_mode=None): body = {"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": name, "namespace": ns, "labels": labels or {}}, "data": data} path = f"/api/v1/namespaces/{ns}/configmaps" @@ -467,7 +644,266 @@ def put_configmap(k8s, ns, name, data, labels=None, resource_version=None): current = k8s.call("GET", f"{path}/{name}", ok404=True) if current is None or is_deleting(current): raise PortalError(f"{name} changed while this request ran (a delete?); request it again") - put_configmap(k8s, ns, name, data, labels, current["metadata"]["resourceVersion"]) + if deployment_mode is not None: + assert_mode_unchanged(read_entry(current), deployment_mode) + if deployment_mode in KUBERNETES_MODES: + raise PortalError("concurrent container registration; refusing to replace its identity") + put_configmap(k8s, ns, name, data, labels, current["metadata"]["resourceVersion"], deployment_mode) + + +CREDENTIAL_WRITER = "saw.redhat.com/credential-writer" +WRITER_NAME = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?") + + +def credential_writer_state(k8s, namespace, holder): + if not WRITER_NAME.fullmatch(holder): + raise PortalError("invalid registration credential writer") + run = k8s.call("GET", f"/apis/tekton.dev/v1/namespaces/{namespace}/pipelineruns/{holder}", ok404=True) + if not run: + return "missing" + terminal = any(c.get("type") == "Succeeded" and c.get("status") in ("True", "False") + for c in run.get("status", {}).get("conditions", [])) + return "incomplete" if terminal else "running" + + +def check_credential_writer(k8s, namespace, cm): + """Serialize Vault changes with registration publication and deletion.""" + holder = (cm or {}).get("metadata", {}).get("annotations", {}).get(CREDENTIAL_WRITER) + if not holder or holder == os.environ.get("PIPELINE_RUN"): + return + state = credential_writer_state(k8s, namespace, holder) + if state == "missing": + raise PortalError("credential writer PipelineRun is missing; operator recovery required") + if state == "running": + raise PortalError("another pipeline is publishing this registration's credentials") + + +def committed_entry(cm): + """Legacy interrupted writers have no proven previous publication.""" + if (cm or {}).get("metadata", {}).get("annotations", {}).get(CREDENTIAL_WRITER): + try: + data = json.loads(cm.get("data", {}).get("committed.json", "{}")) + except (TypeError, ValueError): + raise PortalError("invalid committed registration") from None + if not isinstance(data, dict): + raise PortalError("invalid committed registration") + return data + return read_entry(cm) + + +def reserve_credentials(k8s, ns, name, previous, entry, user): + """A failed/crashed write always has a registered generation for recovery.""" + writer = env("PIPELINE_RUN") + if not WRITER_NAME.fullmatch(writer): + raise PortalError("credential publication requires a named PipelineRun") + path = f"/api/v1/namespaces/{ns}/configmaps" + body = {"apiVersion": "v1", "kind": "ConfigMap", + "metadata": {"name": name, "namespace": ns, + "labels": {WORKSPACE_LABEL: "true", "openshell.pattern/owner": user}, + "annotations": {CREDENTIAL_WRITER: writer}}, + "data": {"user.json": json.dumps(entry, sort_keys=True)}} + if previous: + committed = committed_entry(previous) + if committed: + body["data"]["committed.json"] = json.dumps(committed, sort_keys=True) + meta = body["metadata"] = dict(previous["metadata"]) + meta["annotations"] = {**meta.get("annotations", {}), CREDENTIAL_WRITER: writer} + try: + return k8s.call("PUT" if previous else "POST", path + ("/"+name if previous else ""), body) + except HttpError as exc: + if exc.code != 409: + raise + raise PortalError("concurrent registration changed before credential publication") from None + + +def check_container_identity(user, site): + """Resolve the authenticated catalog login in the configured issuer before any write.""" + if not site or not site.get("subject"): + raise PortalError("missing enrolled container owner") + issuer = env("KEYCLOAK_URL").rstrip("/") + "/realms/" + env("KEYCLOAK_REALM") + relay_issuer = site.get("automaticEnrollment", {}).get("relay", {}).get("issuer") + if relay_issuer and relay_issuer.rstrip("/") != issuer: + raise PortalError("workspace relay and owner directory must use the same issuer") + path = env("KEYCLOAK_SECRET_FILE") + with open(path, encoding="utf-8") as credential: + ids = keycloak_user_ids(env("KEYCLOAK_URL"), env("KEYCLOAK_REALM"), env("KEYCLOAK_CLIENT_ID"), + credential.read().strip(), [user], insecure=False) + if ids.get(user) != site["subject"]: + raise PortalError("current identity does not match the enrolled container owner") + + +def _approved_profile_reference(site, profile): + boundaries = site.get("profileBoundaries", {}) if isinstance(site, dict) else {} + boundary = boundaries.get(profile) if isinstance(boundaries, dict) else None + if isinstance(boundary, dict) and isinstance(boundary.get("reference"), dict): + return boundary["reference"] + automatic = site.get("automaticEnrollment", {}) if isinstance(site, dict) else {} + return automatic.get("reference") if isinstance(automatic, dict) else None + + +def check_live_boundary_approval(k8s, user, site, mode, profile, catalog): + """Check the existing namespace policy against the owner-approved hash.""" + automatic = site.get("automaticEnrollment", {}) if isinstance(site, dict) else {} + if not isinstance(automatic, dict) or not automatic or automatic.get("enabled") is False: + return + reference = _approved_profile_reference(site, profile) + try: + _preflight.verify_boundary_reference(reference, mode) + except _preflight.PreflightError as exc: + raise PortalError("Workspace preflight failed: " + exc.reason) from None + namespace_name = "saw-" + user + try: + namespace = k8s.call("GET", "/api/v1/namespaces/" + namespace_name, ok404=True) + except Exception: + raise PortalError("Workspace preflight failed: BoundaryNamespaceUnavailable") from None + if namespace is None: + # A first registration has no generated namespace policies yet. The + # site reference is complete and valid; shared live prerequisites were + # checked above. Compare exact policy hashes after Argo creates them. + return + labels = namespace.get("metadata", {}).get("labels", {}) + if labels.get(PORTAL_NS_LABEL) != "true": + # check_namespace_is_ours reports this case before any credential write. + return + registration = labels.get("saw.redhat.com/registration", "") + owner_id = labels.get("saw.redhat.com/runtime-owner", "") + if (not GENERATION_RE.fullmatch(registration) + or owner_id != installation_id(user, registration)): + raise PortalError("Workspace preflight failed: BoundaryApprovalIdentityMismatch") + profiles = (catalog.get("profiles", catalog) if isinstance(catalog, dict) else {}) + profile_data = profiles.get(profile) if isinstance(profiles, dict) else None + if not isinstance(profile_data, dict): + raise PortalError("Workspace preflight failed: BoundaryApprovalInvalid") + slots = sorted(sandbox_targets(user, registration, profile_data)) + if not slots: + raise PortalError("Workspace preflight failed: BoundaryApprovalInvalid") + + def resource(path, key, normalize=False): + try: + obj = k8s.call("GET", path, ok404=True) + except Exception: + raise PortalError("Workspace preflight failed: BoundaryPolicyUnavailable") from None + if not isinstance(obj, dict) or not isinstance(obj.get("spec"), dict): + raise PortalError("Workspace preflight failed: BoundaryPolicyUnavailable") + spec = obj["spec"] + if normalize: + digest = _preflight.validation_policy_digest( + spec, user, registration, owner_id, slots) + else: + digest = _preflight.fingerprint(spec) + try: + _preflight.verify_boundary_hash(reference, key, digest) + except _preflight.PreflightError as exc: + raise PortalError("Workspace preflight failed: " + exc.reason + ":" + key) from None + if obj.get("kind") == "ValidatingAdmissionPolicy": + status = obj.get("status", {}) + meta = obj.get("metadata", {}) + if (status.get("observedGeneration") != meta.get("generation") + or "typeChecking" not in status + or status["typeChecking"].get("expressionWarnings")): + raise PortalError("Workspace preflight failed: AdmissionUnavailable:" + key) + + admission = "/apis/admissionregistration.k8s.io/v1" + policy = "saw-" + namespace_name + "-runtime" + resource(admission + "/validatingadmissionpolicies/" + policy, + "validation", normalize=True) + resource(admission + "/validatingadmissionpolicybindings/" + policy, "binding") + if mode == "kata": + resource(admission + "/validatingadmissionpolicies/" + policy + "-kata", + "sandboxPlacement") + resource(admission + "/validatingadmissionpolicybindings/" + policy + "-kata", + "sandboxPlacementBinding") + + +def check_workspace_preflight(k8s, user, site, mode, profile, catalog): + """Run read-only platform gates before reserving or writing keys.""" + observations = _preflight.configuration_facts(site) + observations["api"] = True + try: + cluster_version = k8s.call("GET", "/apis/config.openshift.io/v1/clusterversions/version") + except Exception: + raise PortalError("Workspace preflight failed: ClusterVersionUnavailable") from None + try: + _preflight.verify_cluster_version(cluster_version) + observations["version"] = True + except _preflight.PreflightError as exc: + raise PortalError("Workspace preflight failed: " + exc.reason) from None + crd_names = ("applications.argoproj.io", "applicationsets.argoproj.io", + "sandboxes.agents.x-k8s.io", "externalsecrets.external-secrets.io", + "adminnetworkpolicies.policy.networking.k8s.io") + api_readiness = {} + try: + for crd_name in crd_names: + api_readiness[crd_name] = _preflight.crd_established(k8s.call( + "GET", "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + crd_name, + ok404=True)) + admission_discovery = k8s.call("GET", "/apis/admissionregistration.k8s.io/v1", ok404=True) + api_readiness["admissionregistration.k8s.io/v1"] = ( + _preflight.api_resources_available( + admission_discovery, "admissionregistration.k8s.io/v1", + ("validatingadmissionpolicies", "validatingadmissionpolicybindings"))) + except Exception: + raise PortalError("Workspace preflight failed: PlatformApiUnavailable") from None + observations["agentSandbox"] = api_readiness["sandboxes.agents.x-k8s.io"] + observations["admission"] = api_readiness["admissionregistration.k8s.io/v1"] + observations["network"] = (observations.get("network") is True + and api_readiness["adminnetworkpolicies.policy.networking.k8s.io"]) + try: + storage = k8s.call("GET", "/apis/storage.k8s.io/v1/storageclasses").get("items", []) + _preflight.verify_storage_classes(site, storage) + observations["storage"] = True + except _preflight.PreflightError: + observations["storage"] = False + except Exception: + observations["storage"] = False + try: + store_items = k8s.call("GET", "/apis/external-secrets.io/v1/clustersecretstores").get("items", []) + _preflight.verify_secret_store(site, store_items) + except Exception: + observations["secrets"] = False + try: + vault = Http(env("VAULT_ADDR"), cafile=os.environ.get("VAULT_CACERT") or None, + insecure=os.environ.get("VAULT_SKIP_VERIFY") == "true") + health = vault.call("GET", "/v1/sys/health?standbyok=true&perfstandbyok=true") + observations["secrets"] = (observations.get("secrets") is not False + and api_readiness["externalsecrets.external-secrets.io"] + and _preflight.vault_health_ready(health)) + except Exception: + observations["secrets"] = False + try: + argo = env("ARGO_NAMESPACE") + k8s.call("GET", f"/apis/argoproj.io/v1alpha1/namespaces/{argo}/applications/saw-{user}", ok404=True) + observations["argo"] = (api_readiness["applications.argoproj.io"] + and api_readiness["applicationsets.argoproj.io"]) + except Exception: + observations["argo"] = False + if mode == "kata": + approval = site.get("kataRuntimeClass") if isinstance(site, dict) else None + if not isinstance(approval, dict) or not approval.get("name"): + raise PortalError("Kata preflight failed: RuntimeClassApprovalMissing") + try: + runtime_class = k8s.call("GET", "/apis/node.k8s.io/v1/runtimeclasses/" + approval["name"]) + configs = k8s.call("GET", "/apis/kataconfiguration.openshift.io/v1/kataconfigs").get("items", []) + nodes = k8s.call("GET", "/api/v1/nodes").get("items", []) + kata_crd = "kataconfigs.kataconfiguration.openshift.io" + api_readiness[kata_crd] = _preflight.crd_established(k8s.call( + "GET", "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + kata_crd, + ok404=True)) + _preflight.verify_kata_platform(site, runtime_class, configs, nodes, + api_readiness, cluster_version) + observations.update(osc=True, kataConfig=True, workers=True, runtimeClass=True) + except _preflight.PreflightError as exc: + raise PortalError("Kata preflight failed: " + exc.reason) from None + except Exception: + raise PortalError("Kata preflight failed: KataPlatformUnavailable") from None + try: + _preflight.check_observations(mode, observations) + except _preflight.PreflightError as exc: + raise PortalError("Workspace preflight failed: " + exc.reason) from None + except Exception: + # API errors can contain request details. Keep the pipeline log safe. + raise PortalError("Workspace preflight failed: PlatformApiUnavailable") from None + check_live_boundary_approval(k8s, user, site, mode, profile, catalog) def is_deleting(cm): @@ -504,8 +940,25 @@ def list_workspaces(k8s, ns, deleting=False): log(f"ERROR: registry entry {name} is malformed (no valid user.json); skipped") bad.append(name) continue + if cm["metadata"].get("annotations", {}).get(CREDENTIAL_WRITER) and not deleting: + try: + entry = committed_entry(cm) + if entry and name != f"saw-ws-{entry.get('name')}": + raise PortalError("committed registration has a different owner") + except PortalError: + log(f"ERROR: registry entry {name} has an invalid committed registration; skipped") + bad.append(name) + continue + if not entry: + continue if (cm["metadata"].get("labels") or {}).get(DELETING_LABEL) == "true" and not deleting: continue + try: + entry_deployment_mode(entry) + except PortalError: + log(f"ERROR: registry entry {name} has an invalid deployment mode; skipped") + bad.append(name) + continue out.append(entry) BAD_ENTRIES[:] = sorted(bad) return sorted(out, key=lambda e: e["name"]) @@ -520,14 +973,7 @@ def ui_links(user, profiles, catalog, domain): def sandbox_ui_links(user, profiles, catalog, domain): - out = [] - for profile in profiles: - for w in catalog.get(profile, {}).get("workspaces", []): - for sb in w.get("sandboxes", []): - if w.get("enabled") and sb.get("enabled") and sb.get("uiRoute"): - out.append((f"{sb['name']} UI ({w['name']})", - f"https://{user}-{w['name']}-{sb['name']}-ui.apps.{domain}")) - return out + return _workspace.sandbox_ui_links(user, profiles, catalog, domain) # Groups for RHDH RBAC, so a user sees only the action that applies to them: @@ -553,7 +999,41 @@ def group(name, title, members): return docs -def entities_yaml(workspaces, catalog, domain, rhdh_url, statuses=None, portal_ns="saw-portal", users=None): +def linked_runtime_phase(k8s, link): + """Read only the installation's two named objects; a stale generation cannot be Ready.""" + ref = link["runtime"] + ns = ref["namespace"] + if not all(re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", ref[key]) + for key in ("namespace", "intent", "status")): + raise PortalError("invalid runtime status reference") + def read(name, key): + cm = k8s.call("GET", f"/api/v1/namespaces/{ns}/configmaps/{name}", ok404=True) + if not cm or cm.get("metadata", {}).get("deletionTimestamp"): + return {}, {} + return json.loads(cm.get("data", {}).get(key, "{}")), cm["metadata"].get("labels", {}) + try: + intent, il = read(ref["intent"], "intent.json") + status, sl = read(ref["status"], "status.json") + owner_label = "saw.redhat.com/runtime-owner" + binding = intent.get("bindings", {}) + if (il.get(owner_label) != ref["installationID"] or sl.get(owner_label) != ref["installationID"] + or sl.get("saw.redhat.com/managed-by") != "kubernetes-runtime-controller" + or binding.get("ownerID") != ref["installationID"] or binding.get("ownerSubject") != ref["ownerSubject"] + or binding.get("namespace") != ns or binding.get("mode") != ref["mode"]): + return "Unknown" + if type(intent.get("generation")) is not int or status.get("observedGeneration") != intent["generation"]: + return "Reconciling" + phase = status.get("phase") + if phase == "GenericReady" and intent.get("desiredState") == "Running": + return "Running" + if phase == "Stopped" and intent.get("desiredState") == "Stopped": + return "Stopped" + return "Blocked" if phase == "Blocked" else "Reconciling" + except (PortalError, ValueError, TypeError, AttributeError): + return "Unknown" + + +def entities_yaml(workspaces, catalog, domain, rhdh_url, statuses=None, portal_ns="saw-portal", users=None, runtime_reader=None): """RHDH catalog entities: one Component per workspace (the kind RHDH's catalog lists first, and the one its CI tab is made for), owned by its user, linking the OpenShell web UI and each sandbox UI route, with the @@ -563,7 +1043,7 @@ def entities_yaml(workspaces, catalog, domain, rhdh_url, statuses=None, portal_n for ws in workspaces: user = ws["name"] links = [{"url": url, "title": title, "icon": "dashboard" if i == 0 else "web"} - for i, (title, url) in enumerate(ui_links(user, ws.get("profiles", []), catalog, domain))] + for i, (title, url) in enumerate(registered_links(ws, catalog, domain))] if rhdh_url: # The delete form, with this workspace already chosen. form = urllib.parse.quote(json.dumps({"workspace": f"component:default/saw-{user}"}, @@ -572,6 +1052,7 @@ def entities_yaml(workspaces, catalog, domain, rhdh_url, statuses=None, portal_n "title": "Delete workspace", "icon": "delete"}) description = "Secure Agent Workspace (profiles: " + ", ".join(ws.get("profiles", [])) + ")" annotations = {"openshell.pattern/namespace": f"saw-{user}", + "openshell.pattern/deployment-mode": entry_deployment_mode(ws), # The Tekton tab: the create and delete pipeline runs # labelled with this id in the portal namespace. "backstage.io/kubernetes-id": f"saw-{user}", @@ -589,6 +1070,25 @@ def entities_yaml(workspaces, catalog, domain, rhdh_url, statuses=None, portal_n "links": links}, "spec": {"type": "agent-workspace", "owner": f"user:default/{user}", "lifecycle": "production"}}) + # Operator-linked workspaces have their own lifecycle outside the portal registry. + # Publish only a catalog link. Existing create/delete templates still own VM entries. + for linked in json.loads(os.environ.get("SAW_LINKED_WORKSPACES", "[]")): + owner, slug = linked["owner"], linked["routeLabel"] + check_user(owner) + if not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", slug) or not domain: + raise PortalError("linked workspace requires an ingress DNS label and cluster domain") + runtime_metadata = {} + if "runtime" in linked: + mode = linked["runtime"]["mode"] + phase = linked_runtime_phase(runtime_reader, linked) if runtime_reader else "Unknown" + runtime_metadata = {"annotations": {"openshell.pattern/deployment-mode": mode, "openshell.pattern/status": phase}, + "description": f"OpenShell {mode} workspace. Runtime status: {phase}."} + docs.append({"apiVersion": "backstage.io/v1alpha1", "kind": "Component", + "metadata": {"name": f"linked-{slug}", "title": linked["title"], **runtime_metadata, + "links": [{"url": f"https://{slug}.apps.{domain}", + "title": "OpenClaw UI", "icon": "web"}]}, + "spec": {"type": "agent-workspace", "owner": f"user:default/{owner}", + "lifecycle": "experimental"}}) docs += audience_groups([ws["name"] for ws in workspaces], users) docs.append(get_started(rhdh_url)) return "\n---\n".join(json.dumps(d, indent=2, sort_keys=True) for d in docs) + "\n" @@ -696,7 +1196,7 @@ class OwnerSubjects: Without Keycloak settings (RBAC off) entries stay as they are.""" def __init__(self, ttl=60): - self.ttl, self.ids, self.tried = ttl, {}, {} + self.ttl, self.ids, self.tried, self.failed = ttl, {}, {}, set() def fill(self, workspaces): url = os.environ.get("KEYCLOAK_URL", "") @@ -716,10 +1216,17 @@ def fill(self, workspaces): for name in missing: self.ids.pop(name, None) self.ids.update(found) + self.failed.difference_update(missing) except (PortalError, OSError, ValueError, KeyError) as exc: - log(f"WARN: Keycloak ids not read ({exc}); {', '.join(missing)} not members yet") + log(f"WARN: Keycloak owner lookup failed ({exc}); projection held") for name in missing: self.ids.pop(name, None) + self.failed.update(missing) + # An unavailable identity service is not an empty or disabled owner. + # Hold the entire VM/container projection, including during the retry + # backoff, rather than silently omitting entries or clearing membership. + if any(not w.get("ownerSubject") and w["name"] in self.failed for w in workspaces): + raise PortalError("Keycloak owner lookup unavailable; workspace projection held") return [{**w, "ownerSubject": self.ids[w["name"]]} if not w.get("ownerSubject") and w["name"] in self.ids else w for w in workspaces] @@ -769,6 +1276,14 @@ def ping_url(url, timeout=3): return False +def owner_route_available(url, timeout=3): + issuer = os.environ.get("KEYCLOAK_URL", "").rstrip("/") + if not issuer: + return False + issuer += "/realms/" + os.environ.get("KEYCLOAK_REALM", "openshell") + return _workspace.owner_route_available(url, issuer, urllib.request.build_opener(NoRedirect()), timeout) + + def app_summary(app): st = app.get("status") or {} sync = (st.get("sync") or {}).get("status") or "Unknown" @@ -782,7 +1297,7 @@ def workspace_status(k8s, user, catalog, ns, argo_ns, domain, ping=None): `phase` is the last stage reached in order ("none" before the registry entry exists, "failed" when a stage failed); `text` is a short report for the template's log and output.""" - ping = ping or ping_url + vm_ping = ping or ping_url reached, details, failure = {}, {}, "" cm = k8s.call("GET", f"/api/v1/namespaces/{ns}/configmaps/saw-ws-{user}", ok404=True) try: @@ -791,10 +1306,34 @@ def workspace_status(k8s, user, catalog, ns, argo_ns, domain, ping=None): entry = {} profiles = entry.get("profiles", []) if isinstance(entry, dict) else [] if cm is not None and (cm["metadata"].get("labels") or {}).get(DELETING_LABEL) == "true": - gone, detail = removal_status(k8s, user, argo_ns) + try: + _, detail = removal_status(k8s, user, argo_ns, entry) + except HttpError as exc: + if exc.code != 403: + raise + # The read-only catalog has no cluster namespace inventory. Only + # wait-gone, running as the provisioner, can attest full removal. + detail = "The deletion pipeline is verifying workspace removal" return {"user": user, "phase": "deleting", "title": PHASE_TITLES["deleting"], "message": detail, "ready": False, "failed": False, "steps": [], "links": [], "text": f"Deleting: {detail}"} + holder = (cm or {}).get("metadata", {}).get("annotations", {}).get(CREDENTIAL_WRITER) + if holder: + state = credential_writer_state(k8s, ns, holder) + message = {"running": "Publishing credentials", + "incomplete": "Credential publication incomplete. Retry the create form or delete the workspace.", + "missing": "Credential writer is missing. Ask the operator to recover the registration."}[state] + return {"user": user, "deploymentMode": entry_deployment_mode(entry), "phase": "publishing" if state == "running" else "failed", + "title": message, "text": message, "message": message, "ready": False, + "failed": state != "running", "steps": [], "links": []} + mode = entry_deployment_mode(entry) + if mode in KUBERNETES_MODES and container_site(user): + return container_status(k8s, user, entry, argo_ns, domain, ping or owner_route_available, catalog) + if mode not in PROVISIONABLE_MODES: + message = f"{mode} workspace lifecycle and readiness are not qualified in this release" + return {"user": user, "deploymentMode": mode, "phase": "runtime-not-qualified", + "title": "Runtime not available", "message": message, "ready": False, + "failed": True, "steps": [], "links": [], "text": message} reached["registered"] = cm is not None details["registered"] = (f"ConfigMap saw-ws-{user}, profile {', '.join(profiles)}" if cm else f"no registry entry saw-ws-{user} yet") @@ -832,7 +1371,7 @@ def workspace_status(k8s, user, catalog, ns, argo_ns, domain, ping=None): reached["ready"] = False details["ready"] = "installing; waiting for workspace routes" else: - down = [title for title, url in uis if not ping(url)] + down = [title for title, url in uis if not vm_ping(url)] reached["ready"] = not down details["ready"] = ("UIs answer: " + ", ".join(t for t, _ in uis) if not down else "installing; waiting for " + ", ".join(down)) @@ -865,7 +1404,7 @@ def workspace_status(k8s, user, catalog, ns, argo_ns, domain, ping=None): "text": "\n".join(lines)} -def removal_status(k8s, user, argo_ns): +def removal_status(k8s, user, argo_ns, entry=None): """(gone, what is left) of a workspace being deleted: its Argo CD applications and, with pruneOnRemove, namespace saw- (without it the namespace and the VM stay on purpose).""" @@ -876,6 +1415,16 @@ def removal_status(k8s, user, argo_ns): ns = k8s.call("GET", f"/api/v1/namespaces/saw-{user}", ok404=True) if prune else None if ns is not None: left.append(f"namespace saw-{user} ({(ns.get('status') or {}).get('phase', 'Active')})") + if entry and entry_deployment_mode(entry) in KUBERNETES_MODES: + reg = entry_generation(user, entry) + if not reg: + raise PortalError("refusing container cleanup without a registration") + ui_names=entry.get("uiNamespaces",[f"saw-ui-{reg}"]) + if any(not re.fullmatch("saw-ui-[a-f0-9]{16}", n) for n in ui_names): + raise PortalError("invalid portal UI namespace reference") + for scoped in [f"saw-ctl-{reg}", *ui_names]: + if k8s.call("GET", "/api/v1/namespaces/" + scoped, ok404=True) is not None: + left.append("namespace " + scoped) return not left, ("removed" if not left else "Argo CD is removing " + ", ".join(left)) @@ -944,20 +1493,31 @@ def wait_for(check, done, seconds, sleep=time.sleep, clock=time.monotonic): class JwksCache: - """RHDH's JWKS documents, refetched every few minutes.""" + """Cache RHDH keys, with bounded on-demand refetches for rotating key IDs.""" def __init__(self, ttl=300): self.ttl, self.docs = ttl, {} - - def __call__(self, path): - now = time.time() - hit = self.docs.get(path) - if hit and now - hit[0] < self.ttl: - return hit[1] - rhdh = env("RHDH_INTERNAL_URL").rstrip("/") - doc = fetch_json(rhdh + path, insecure=os.environ.get("RHDH_SKIP_VERIFY") == "true") - self.docs[path] = (now, doc) - return doc + self.next_miss_refresh = {} + self.lock = threading.Lock() + + def __call__(self, path, kid=None): + # The verifier chooses one of two fixed paths. No token field selects + # the issuer or URL. Coalesce misses per issuer, not per untrusted kid. + with self.lock: + now = time.monotonic() + hit = self.docs.get(path) + if hit and now - hit[0] < self.ttl: + if kid is None or any(k.get("kid") == kid for k in hit[1].get("keys", [])): + return hit[1] + if now < self.next_miss_refresh.get(path, 0): + return hit[1] + self.next_miss_refresh[path] = now + 5 + rhdh = env("RHDH_INTERNAL_URL").rstrip("/") + doc = fetch_json(rhdh + path) + self.docs[path] = (time.monotonic(), doc) + if kid is not None and not any(k.get("kid") == kid for k in doc.get("keys", [])): + self.next_miss_refresh[path] = time.monotonic() + 5 + return doc PORTAL_NS_LABEL = "saw.redhat.com/portal" @@ -1072,25 +1632,110 @@ def handle(action, request_name): name = f"saw-ws-{user}" cm = k8s.call("GET", f"/api/v1/namespaces/{ns}/configmaps/{name}", ok404=True) deleting = cm is not None and is_deleting(cm) - if action == "create": + mode = entry_deployment_mode(read_entry(cm)) if cm else None + if mode in KUBERNETES_MODES or (action == "create" and data.get("deploymentMode") in KUBERNETES_MODES): + check_container_identity(user, container_site(user)) + if caller != user: + pins = json.loads(os.environ.get("SAW_PORTAL_ADMIN_SUBJECTS", "{}")) + check_container_identity(caller, {"subject": pins.get(caller)}) + check_credential_writer(k8s, ns, cm) + if action == "create" and "runtimeAction" in data: + if (cm or {}).get("metadata", {}).get("annotations", {}).get(CREDENTIAL_WRITER): + raise PortalError("finish credential publication before changing the runtime state") + entry = read_entry(cm) + operation = data["runtimeAction"] + if (operation not in ("start", "stop") or caller != user or deleting or not cm + or entry_deployment_mode(entry) not in KUBERNETES_MODES or entry.get("lifecycleVersion") != 1 + or entry_generation(user, entry) != data.get("registration") + or f"g-{entry.get('intentGeneration')}" != data.get("expectedGeneration")): + raise PortalError("lifecycle request does not match the current owner registration and generation") + desired = "Running" if operation == "start" else "Stopped" + if entry["desiredState"] != desired: + entry["desiredState"] = desired + entry["intentGeneration"] += 1 + cm["data"]["user.json"] = json.dumps(entry, sort_keys=True) + k8s.call("PUT", f"/api/v1/namespaces/{ns}/configmaps/{name}", cm) + log(f"workspace {user} requested {desired}") + elif action == "create": if deleting: raise PortalError(f"workspace saw-{user} is being deleted; request it again when " "the delete pipeline has finished") - profile, secrets = parse_request(data, catalog) + mode = request_deployment_mode(data) + site = container_site(user) if mode in KUBERNETES_MODES else None + profile = None + secrets = None + if mode in KUBERNETES_MODES: + profile, secrets = parse_request(data, catalog) + check_workspace_preflight(k8s, user, site, mode, profile, catalog) + if cm is not None: + assert_mode_unchanged(read_entry(cm), mode) + if mode in KUBERNETES_MODES and site: + try: + _workspace.validate_runtime_placement(mode, site) + except _workspace.WorkspaceError as exc: + raise PortalError(str(exc)) from exc + approved = container_profiles(site) + if data.get("profile") not in approved: + raise PortalError("selected profile is not approved for your container workspace; " + "approved profiles: " + (", ".join(sorted(approved)) or "none") + + ". Ask the site administrator to enable another profile") + if os.environ.get("PRUNE_ON_REMOVE", "true") != "true": + raise PortalError("container provisioning requires foreground cleanup; " + "the site administrator must enable portal.pruneOnRemove") + try: + _preflight.verify_profile_image_access(site, profile, catalog) + except _preflight.PreflightError as exc: + raise PortalError("Workspace preflight failed: " + exc.reason) from None + elif mode not in PROVISIONABLE_MODES: + raise PortalError(f"{mode} workspace provisioning is not available in this release") + if profile is None: + profile, secrets = parse_request(data, catalog) + if mode in KUBERNETES_MODES: + if cm and read_entry(cm)["profiles"] != [profile]: + raise PortalError("changing the profile requires a new container registration") + endpoint = approved_endpoint(catalog[profile], secrets, + site.get("profileBoundaries", {}).get(profile, {})) + if cm and endpoint != read_entry(cm).get("customInferenceURL"): + raise PortalError("changing the endpoint requires a new registration") + models = public_model_inputs(catalog[profile], secrets) + previous = committed_entry(cm) + lifecycle = previous.get("lifecycleVersion") == 1 if previous else container_site(user).get("modelLifecycle") is True + if previous and not lifecycle and previous.get("modelInputs", {}) != models: + raise PortalError("changing model selection requires a new container registration") check_namespace_is_ours(k8s, user) check_applications_are_ours(k8s, user) # An update keeps the entry's keys where they are; a new # registration gets its own generation (see vault_prefix). generation = entry_generation(user, read_entry(cm)) if cm else new_generation() prefix = vault_prefix(user, generation) + entry = registry_entry(user, profile, generation, mode) + if mode in KUBERNETES_MODES: + if lifecycle: + entry["lifecycleVersion"] = 1 + entry["initialModelInputs"] = previous.get("initialModelInputs", models) + entry["desiredState"] = previous.get("desiredState", "Running") + changed = bool(previous and previous.get("modelInputs", {}) != models) + entry["restartGeneration"] = (previous["intentGeneration"] + 1 if changed + else previous.get("restartGeneration", 0)) + if endpoint: + entry["customInferenceURL"] = endpoint + if "dedicatedEndpointSecrets" in catalog[profile]: + entry["webSearch"] = {"enabled": False} + if models: + entry["modelInputs"] = models + if "approvedProfiles" in site.get("automaticEnrollment",{}): + slots=sandbox_targets(user,generation,catalog[profile]) + entry["uiNamespaces"]=sorted({"saw-ui-"+slot["relayID"] for slot in slots.values() if slot["ui"]}) + entry["intentGeneration"] = previous.get("intentGeneration", 0) + 1 + cm = reserve_credentials(k8s, ns, name, cm, entry, user) vault = vault_client() for secret, values in secrets.items(): vault.write(f"{prefix}/{secret}", values) log(f"Vault: {prefix}/{secret} ({', '.join(sorted(values))})") - entry = registry_entry(user, profile, generation) put_configmap(k8s, ns, name, {"user.json": json.dumps(entry, sort_keys=True)}, {WORKSPACE_LABEL: "true", "openshell.pattern/owner": user}, - resource_version=cm["metadata"].get("resourceVersion") if cm else None) + resource_version=cm["metadata"].get("resourceVersion") if cm else None, + deployment_mode=mode) log(f"workspace saw-{user} registered with profile {profile}; Argo CD builds it next") else: argo_app = f"/apis/argoproj.io/v1alpha1/namespaces/{env('ARGO_NAMESPACE')}/applications/portal-ws-{user}" @@ -1197,13 +1842,18 @@ def wait_stage(stage, user, timeout, sleep=time.sleep, clock=time.monotonic, pin domain = os.environ.get("CLUSTER_DOMAIN", "") if stage != "gone" and stage not in STAGE_IDS: raise PortalError(f"unknown stage {stage}; one of {', '.join(STAGE_IDS)}, gone") - deadline, last = clock() + timeout, None + started, last = clock(), None while True: + budget = timeout if stage == "gone": - done, detail = removal_status(k8s, user, argo_ns) + marker = k8s.call("GET", f"/api/v1/namespaces/{ns}/configmaps/saw-ws-{user}", ok404=True) + done, detail = removal_status(k8s, user, argo_ns, read_entry(marker)) failed = False else: st = workspace_status(k8s, user, catalog, ns, argo_ns, domain, ping) + if stage == "vm" and st.get("deploymentMode") in KUBERNETES_MODES: + # The initial intent can only be published after the new registration is reviewed. + budget = max(timeout, int(os.environ.get("SAW_CONTAINER_ENROLLMENT_SECONDS", timeout))) current = next((x for x in st["steps"] if x["id"] == stage), {}) detail = current.get("detail") or st["message"] done = current.get("state") == "done" @@ -1218,8 +1868,8 @@ def wait_stage(stage, user, timeout, sleep=time.sleep, clock=time.monotonic, pin return if failed: raise PortalError(detail) - if clock() >= deadline: - raise PortalError(f"not done after {timeout // 60} minutes: {detail}") + if clock() >= started + budget: + raise PortalError(f"not done after {budget // 60} minutes: {detail}") sleep(POLL_SECONDS * 2) @@ -1243,16 +1893,72 @@ def cleanup(max_age=REQUEST_MAX_AGE): # -- ApplicationSet plugin generator -------------------------------------------- -def generator_params(workspaces, defaults): - """One parameter set per workspace: `name`, and `values`, the saw-users - chart values for that user alone (JSON, which is YAML).""" + +def generator_params(workspaces, defaults, read=None, catalog=None, previous=None): + """Adapt authenticated registry entries to the common workspace projector.""" out = [] - for ws in workspaces: - values = {**defaults, "users": [ws]} - out.append({"name": ws["name"], "values": json.dumps(values, sort_keys=True)}) + for entry in workspaces: + ws = {**entry, "values": {**entry.get("values", {}), + "deployment": {"mode": entry_deployment_mode(entry)}}} + site = container_site(ws["name"]) if entry_deployment_mode(ws) in KUBERNETES_MODES else {} + if site: + if not ws.get("ownerSubject") or ws["ownerSubject"] != site["subject"]: + continue + reg = entry_generation(ws["name"], ws) + ws.update(registration=reg, installationID=installation_id(ws["name"], reg)) + def observed(path): + try: + return read(path) + except HttpError as exc: + if exc.code == 403: # named reader grant is not installed yet + return {} + raise + try: + values = _workspace.project_workspace(ws, defaults, site, catalog or {}, + observed if read else None, previous) + except _workspace.WorkspaceError as exc: + raise PortalError(str(exc)) from exc + out.append({"name": ws["name"], "revision": values.get("global", {}).get("targetRevision", ""), + "values": json.dumps(values, sort_keys=True)}) return out +class CatalogRouteCache: + """A catalog read never waits for an endpoint. Explicit owner checks bypass this cache.""" + def __init__(self, workers=4, capacity=512, ttl=30): + self.pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="catalog-route") + self.capacity, self.ttl = capacity, ttl + self.lock = threading.Lock() + self.entries = {} + + def probe(self, scope, url, check): + key = (*scope, url) + now = time.monotonic() + with self.lock: + current = self.entries.get(key) + if current: + if not current.done(): + return False + finished, ready = current.result() + if now - finished < self.ttl: + return ready + self.entries = {k: f for k, f in self.entries.items() + if not f.done() or now - f.result()[0] < self.ttl} + if len(self.entries) >= self.capacity: + return False + future = self.pool.submit(self._check, check, url) + self.entries[key] = future + return False + + @staticmethod + def _check(check, url): + try: + ready = check(url) + except (ValueError, OSError): + ready = False + return time.monotonic(), ready + + def serve(): token = open(env("GENERATOR_TOKEN_FILE"), encoding="utf-8").read().strip() ns = env("NAMESPACE") @@ -1263,6 +1969,7 @@ def serve(): jwks = JwksCache() realm_users = RealmUsers() owner_subjects = OwnerSubjects() + route_cache = CatalogRouteCache() class Handler(http.server.BaseHTTPRequestHandler): def _send(self, code, body): @@ -1300,11 +2007,17 @@ def do_GET(self): statuses = {} for ws in workspaces: try: - statuses[ws["name"]] = workspace_status(k8s, ws["name"], catalog, ns, argo_ns, domain) + mode = entry_deployment_mode(ws) + scope = (ws["name"], mode, entry_generation(ws["name"], ws), + ws.get("intentGeneration", 0), os.environ.get("KEYCLOAK_URL"), + os.environ.get("KEYCLOAK_REALM")) + check = owner_route_available if mode in KUBERNETES_MODES else ping_url + statuses[ws["name"]] = workspace_status(k8s, ws["name"], catalog, ns, argo_ns, domain, + ping=lambda url: route_cache.probe(scope, url, check)) except PortalError as exc: log(f"WARN: status of saw-{ws['name']}: {exc}") text = entities_yaml(workspaces, catalog, domain, - os.environ.get("RHDH_BASE_URL", ""), statuses, ns, realm_users()) + os.environ.get("RHDH_BASE_URL", ""), statuses, ns, realm_users(), runtime_reader=k8s) except PortalError as exc: log(f"ERROR: {exc}") return self._send(500, {"error": str(exc)}) @@ -1375,7 +2088,11 @@ def do_POST(self): if self.headers.get("Authorization", "") != f"Bearer {token}": return self._send(403, {"error": "forbidden"}) try: - params = generator_params(owner_subjects.fill(list_workspaces(kube(), ns)), defaults) + params = generator_params(owner_subjects.fill(list_workspaces(kube(), ns)), defaults, + lambda path:kube().call("GET",path,ok404=True),catalog, + lambda owner: _workspace.previous_child(kube().call("GET", + f"/apis/argoproj.io/v1alpha1/namespaces/{argo_ns}/applications/portal-ws-{owner['name']}", + ok404=True), "saw-portal-workspaces", owner)) except PortalError as exc: log(f"ERROR: {exc}") return self._send(500, {"error": str(exc)}) diff --git a/charts/openshell-rhdh/files/preflight.py b/charts/openshell-rhdh/files/preflight.py new file mode 100644 index 000000000..c32862180 --- /dev/null +++ b/charts/openshell-rhdh/files/preflight.py @@ -0,0 +1,477 @@ +"""Shared, read-only checks for SAW Kubernetes workspace provisioning. + +Callers collect facts from their cluster client. This module validates only +public configuration and observed resource metadata. It never reads Secrets. +""" + +import hashlib +import json +import re + + +class PreflightError(ValueError): + """A failed check with a fixed, safe status code.""" + + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +CHECK_REASONS = { + "api": "ApiUnavailable", + "version": "ClusterVersionUnavailable", + "osc": "SandboxedContainersUnavailable", + "kataConfig": "KataConfigNotReady", + "workers": "KataWorkersNotReady", + "runtimeClass": "RuntimeClassRejected", + "agentSandbox": "AgentSandboxUnavailable", + "admission": "AdmissionUnavailable", + "network": "NetworkPolicyRejected", + "argo": "ArgoAccessDenied", + "storage": "StorageUnavailable", + "secrets": "SecretServicesUnavailable", + "registry": "RegistryUnavailable", + "providerEgress": "ProviderEgressRejected", + "revisions": "UnpinnedRevision", +} + +COMMON_CHECKS = ( + "api", "version", "agentSandbox", "admission", "network", "argo", + "storage", "secrets", "registry", "providerEgress", "revisions", +) +KATA_CHECKS = ("osc", "kataConfig", "workers", "runtimeClass") + +# This pinned image has /sandbox mode 0770, owned by 1000:0. The OpenShift +# namespace UID cannot traverse it. The reviewed notebook build adds o+x. +NON_TRAVERSABLE_WORKLOAD_IMAGES = frozenset({ + "quay.io/aipcc/base-images/agentic/openclaw@sha256:" + "896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", +}) + + +def verify_profile_image_access(site, profile_name, catalog): + """Reject known inaccessible workload images before enrollment starts.""" + profile = catalog.get(profile_name) if isinstance(catalog, dict) else None + if not isinstance(profile, dict) or not isinstance(site, dict): + raise PreflightError("WorkloadImageApprovalMissing") + overrides = site.get("profileImageOverrides", {}) + if not isinstance(overrides, dict): + raise PreflightError("WorkloadImageApprovalInvalid") + selected = overrides.get(profile_name, {}) + if not isinstance(selected, dict): + raise PreflightError("WorkloadImageApprovalInvalid") + for workspace in profile.get("workspaces", []): + if not isinstance(workspace, dict) or not workspace.get("enabled"): + continue + for sandbox in workspace.get("sandboxes", []): + if not isinstance(sandbox, dict) or not sandbox.get("enabled"): + continue + image = sandbox.get("image") + if image in NON_TRAVERSABLE_WORKLOAD_IMAGES: + replacement = selected.get(sandbox.get("name"), image) + if replacement == image: + raise PreflightError("WorkloadImageNotNamespaceAccessible") + if (not isinstance(replacement, str) or not re.fullmatch( + r"[a-z0-9][a-z0-9./:_-]+@sha256:[a-f0-9]{64}", replacement)): + raise PreflightError("WorkloadImageApprovalInvalid") + return True + + +def runtime_placement(runtime_class): + """Return the canonical placement payload used by owner approvals.""" + return {key: runtime_class.get(key) for key in ("handler", "overhead", "scheduling")} + + +def runtime_placement_digest(runtime_class): + raw = json.dumps(runtime_placement(runtime_class), sort_keys=True, + separators=(",", ":"), ensure_ascii=True).encode() + return hashlib.sha256(raw).hexdigest() + + +def fingerprint(value): + """Match the canonical JSON hash used by boundary approvals.""" + raw = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(raw.encode()).hexdigest() + + +def validation_policy_digest(spec, owner, registration, installation_id, slot_keys): + """Hash a live workspace policy with only registration names normalized.""" + if (not isinstance(spec, dict) or not isinstance(owner, str) or not owner + or not isinstance(registration, str) or not re.fullmatch(r"[a-f0-9]{16}", registration) + or not isinstance(installation_id, str) or not installation_id + or not isinstance(slot_keys, (list, tuple)) or not slot_keys + or any(not isinstance(slot, str) or not slot for slot in slot_keys) + or len(set(slot_keys)) != len(slot_keys)): + raise PreflightError("BoundaryApprovalInvalid") + text = json.dumps(spec, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + if "@SAW_" in text: + raise PreflightError("BoundaryApprovalInvalid") + text = text.replace("saw-ctl-" + registration, "@SAW_CONTROL_NS@") + text = text.replace(installation_id, "@SAW_INSTALLATION@") + for slot in slot_keys: + relay = (hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] + if len(slot_keys) > 1 else registration) + if len(slot_keys) > 1: + text = text.replace(owner + "-" + relay[:8], "@SAW_SLOT_" + slot + "@") + text = text.replace("gate-" + relay + "-", "gate-@SAW_RELAY_" + slot + "@-") + text = text.replace("ui-" + relay + "-browser-proof", + "ui-@SAW_RELAY_" + slot + "@-browser-proof") + return fingerprint(json.loads(text)) + + +def verify_boundary_hash(reference, key, observed): + """Fail with a fixed reason when a live policy differs from approval.""" + if not isinstance(reference, dict): + raise PreflightError("BoundaryApprovalMissing") + if key not in reference: + raise PreflightError("BoundaryApprovalMissing") + approved = reference.get(key) + if not isinstance(approved, str) or not re.fullmatch(r"[a-f0-9]{64}", approved): + raise PreflightError("BoundaryApprovalInvalid") + if approved != observed: + raise PreflightError("BoundaryApprovalChanged") + return True + + +def verify_boundary_reference(reference, mode): + """Require every policy approval before a new workspace can be reserved.""" + if mode not in {"container", "kata"}: + raise PreflightError("BoundaryApprovalInvalid") + if not isinstance(reference, dict): + raise PreflightError("BoundaryApprovalMissing") + keys = {"validation", "binding"} + if mode == "kata": + keys.update(("sandboxPlacement", "sandboxPlacementBinding")) + for key in keys: + approved = reference.get(key) + if approved is None: + raise PreflightError("BoundaryApprovalMissing") + if not isinstance(approved, str) or not re.fullmatch(r"[a-f0-9]{64}", approved): + raise PreflightError("BoundaryApprovalInvalid") + return True + + +def check_runtime_class(approved, observed): + """Check a RuntimeClass and return fixed, safe failure reasons.""" + if not isinstance(approved, dict) or not approved.get("name"): + raise PreflightError("RuntimeClassApprovalMissing") + if not isinstance(observed, dict): + raise PreflightError("RuntimeClassUnavailable") + metadata = observed.get("metadata") or {} + if not isinstance(metadata, dict): + raise PreflightError("RuntimeClassUnavailable") + if metadata.get("name") != approved["name"]: + raise PreflightError("RuntimeClassNameMismatch") + if not approved.get("uid") or metadata.get("uid") != approved["uid"]: + raise PreflightError("RuntimeClassIdentityMismatch") + if observed.get("handler") != "kata": + raise PreflightError("RuntimeClassHandlerMismatch") + labels = metadata.get("labels") or {} + if not isinstance(labels, dict): + raise PreflightError("RuntimeClassNotAttachable") + if labels.get("openshell.ai/sandbox-attachable") != "true": + raise PreflightError("RuntimeClassNotAttachable") + if not isinstance(observed.get("scheduling"), dict) or not observed["scheduling"]: + raise PreflightError("RuntimeClassSchedulingInvalid") + approved_digest = approved.get("sha256") + if not isinstance(approved_digest, str) or not re.fullmatch(r"[a-f0-9]{64}", approved_digest): + raise PreflightError("RuntimeClassApprovalInvalid") + if runtime_placement_digest(observed) != approved_digest: + raise PreflightError("RuntimeClassPlacementChanged") + return True + + +def check_observations(mode, observations): + """Require a complete fact set. Missing facts fail closed with safe codes. + + Values are booleans produced by a read-only cluster adapter. Do not pass + API responses, exception strings, URLs with credentials, or Secret data. + """ + if mode not in {"container", "kata"} or not isinstance(observations, dict): + raise PreflightError("InvalidPreflightRequest") + required = COMMON_CHECKS + (KATA_CHECKS if mode == "kata" else ()) + for check in required: + if observations.get(check) is not True: + raise PreflightError(CHECK_REASONS[check]) + return tuple(required) + + +def configuration_facts(site): + """Check site-owned inputs that do not require cluster access.""" + if not isinstance(site, dict): + return {key: False for key in ("network", "storage", "secrets", "registry", + "providerEgress", "revisions")} + stack = site.get("stack") if isinstance(site.get("stack"), dict) else {} + revisions = [item.get("revision") for item in stack.values() if isinstance(item, dict)] + required_components = ("identity", "policy", "proxy", "gateway", "audit") + revision_pin = all(isinstance(stack.get(name), dict) for name in required_components) and all( + isinstance(stack[name].get("revision"), str) + and re.fullmatch(r"[a-f0-9]{40}", stack[name]["revision"]) + for name in required_components) and bool(revisions) and len(revisions) == len(stack) and all( + isinstance(value, str) and re.fullmatch(r"[a-f0-9]{40}", value) for value in revisions) + enrollment = site.get("automaticEnrollment") or {} + if enrollment and not isinstance(enrollment, dict): + revision_pin = False + elif enrollment: + revision_pin = revision_pin and bool(re.fullmatch(r"[a-f0-9]{40}", str(enrollment.get("revision", "")))) + + image_values = [] + storage_values = [] + def walk(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + next_path = path + (str(key).lower(),) + if key == "image" and isinstance(child, str) and child: + image_values.append(child) + if key in {"storageClass", "storageClassName"} and isinstance(child, str) and child: + storage_values.append(child) + if key == "className" and "storage" in path and isinstance(child, str) and child: + storage_values.append(child) + walk(child, next_path) + elif isinstance(value, list): + for child in value: + walk(child, path) + walk(site) + + declared_hosts = site.get("providerAllowHosts") + policy = site.get("providerPolicy") + clean_hosts = (isinstance(declared_hosts, list) and bool(declared_hosts) + and all(isinstance(host, str) and host and "*" not in host + for host in declared_hosts) + and (policy is None or (isinstance(policy, dict) + and isinstance(policy.get("allowHosts"), list) + and all(isinstance(host, str) for host in policy["allowHosts"]) + and set(policy["allowHosts"]) == set(declared_hosts)))) + proxy = stack.get("proxy") or {} + proxy_values = proxy.get("values") or {} if isinstance(proxy, dict) else {} + bootstrap_egress = ((proxy_values.get("network") or {}).get("bootstrapEgress") + if isinstance(proxy_values, dict) else None) + storage_ok = bool(storage_values) and all(value.strip() for value in storage_values) + return { + "network": bool(stack.get("policy")) and isinstance(bootstrap_egress, list) + and bool(bootstrap_egress), + "storage": storage_ok, + "secrets": isinstance(site.get("secretStore", "vault-backend"), str) + and bool(site.get("secretStore", "vault-backend")), + "registry": bool(image_values) and all("@sha256:" in image for image in image_values), + "providerEgress": clean_hosts, + "revisions": revision_pin, + } + + +def storage_class_names(site): + names = set() + def walk(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + if key in {"storageClass", "storageClassName"} and isinstance(child, str) and child: + names.add(child) + elif key == "className" and "storage" in path and isinstance(child, str) and child: + names.add(child) + walk(child, path + (str(key).lower(),)) + elif isinstance(value, list): + for child in value: + walk(child, path) + walk(site) + return tuple(sorted(names)) + + +def verify_storage_classes(site, storage_classes): + names = storage_class_names(site) + available = {item.get("metadata", {}).get("name") for item in storage_classes + if isinstance(item, dict)} + default = any((item.get("metadata", {}).get("annotations") or {}).get( + "storageclass.kubernetes.io/is-default-class") == "true" + for item in storage_classes if isinstance(item, dict)) + if names and not set(names).issubset(available): + raise PreflightError("StorageUnavailable") + if not names and not default: + raise PreflightError("StorageUnavailable") + return names + + +def verify_secret_store(site, stores): + name = secret_store_name(site) + for store in stores: + metadata = store.get("metadata") or {} if isinstance(store, dict) else {} + if isinstance(metadata, dict) and metadata.get("name") == name and _ready(store): + return name + raise PreflightError("SecretServicesUnavailable") + + +def secret_store_name(site): + if not isinstance(site, dict): + return "vault-backend" + stack = site.get("stack") if isinstance(site.get("stack"), dict) else {} + identity = stack.get("identity") if isinstance(stack.get("identity"), dict) else {} + identity_values = identity.get("values") if isinstance(identity.get("values"), dict) else {} + return site.get("secretStore") or identity_values.get("secretStore") or "vault-backend" + + +def _ready(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and any( + isinstance(c, dict) and c.get("type") in {"Ready", "Completed"} and c.get("status") == "True" + for c in conditions) + + +def kata_config_ready(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + nodes = status.get("kataNodes") or {} + if not isinstance(nodes, dict): + return False + count = nodes.get("nodeCount") + ready_count = nodes.get("readyNodeCount") + if (type(count) is not int or count < 1 or type(ready_count) is not int + or ready_count != count): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and not any( + isinstance(condition, dict) and condition.get("type") == "InProgress" + and condition.get("status") == "True" for condition in conditions) + + +def kata_config_installed_nodes(obj): + if not isinstance(obj, dict): + return () + status = obj.get("status") or {} + if not isinstance(status, dict): + return () + nodes = status.get("kataNodes") or {} + if not isinstance(nodes, dict): + return () + installed = nodes.get("installed") + if not isinstance(installed, list): + return () + return tuple(name for name in installed if isinstance(name, str) and name) + + +def crd_established(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and any( + isinstance(condition, dict) and condition.get("type") == "Established" + and condition.get("status") == "True" for condition in conditions) + + +def api_resources_available(discovery, group_version, resource_names): + """Check served API resources from discovery, including built-in APIs.""" + if not isinstance(discovery, dict) or discovery.get("groupVersion") != group_version: + return False + resources = discovery.get("resources") + if not isinstance(resources, list): + return False + available = {item.get("name") for item in resources if isinstance(item, dict)} + return set(resource_names).issubset(available) + + +def cluster_version_ready(obj): + try: + verify_cluster_version(obj) + return True + except PreflightError: + return False + + +def verify_cluster_version(obj): + if not isinstance(obj, dict): + raise PreflightError("ClusterVersionUnavailable") + status = obj.get("status") or {} + if not isinstance(status, dict): + raise PreflightError("ClusterVersionUnavailable") + conditions = status.get("conditions") or [] + available = isinstance(conditions, list) and any( + isinstance(condition, dict) and condition.get("type") == "Available" + and condition.get("status") == "True" for condition in conditions) + if not available: + raise PreflightError("ClusterVersionUnavailable") + desired = status.get("desired") or {} + version = desired.get("version") if isinstance(desired, dict) else None + if not isinstance(version, str): + raise PreflightError("ClusterVersionUnavailable") + version = version.strip() + match = re.fullmatch(r"(\d+)\.(\d+)(?:\.\d+)?(?:-[A-Za-z0-9.-]+)?", version) + if not match: + raise PreflightError("ClusterVersionUnavailable") + if tuple(map(int, match.groups())) < (4, 22): + raise PreflightError("UnsupportedOpenShiftVersion") + return version + + +def vault_health_ready(obj): + return isinstance(obj, dict) and obj.get("initialized") is True and obj.get("sealed") is False + + +def _matches_selector(labels, selector): + return bool(selector) and all(labels.get(key) == value for key, value in selector.items()) + + +def verify_kata_platform(site, runtime_class, kata_configs, nodes, api_readiness, + cluster_version): + """Check live Kata prerequisites from read-only Kubernetes API snapshots.""" + verify_cluster_version(cluster_version) + approved = site.get("kataRuntimeClass") if isinstance(site, dict) else None + check_runtime_class(approved, runtime_class) + if not isinstance(api_readiness, dict): + raise PreflightError("AgentSandboxUnavailable") + if api_readiness.get("kataconfigs.kataconfiguration.openshift.io") is not True: + raise PreflightError("SandboxedContainersUnavailable") + if not isinstance(kata_configs, list): + raise PreflightError("KataConfigNotReady") + ready_configs = [item for item in kata_configs + if isinstance(item, dict) and kata_config_ready(item)] + if not ready_configs: + raise PreflightError("KataConfigNotReady") + installed_nodes = {name for item in ready_configs for name in kata_config_installed_nodes(item)} + placement = (runtime_class.get("scheduling") or {}).get("nodeSelector") or {} + if not isinstance(placement, dict) or not placement: + raise PreflightError("RuntimeClassSchedulingInvalid") + eligible = [] + if not isinstance(nodes, list): + raise PreflightError("KataWorkersNotReady") + for node in nodes: + if not isinstance(node, dict): + continue + metadata = node.get("metadata") or {} + if not isinstance(metadata, dict): + continue + labels = metadata.get("labels") or {} + if not isinstance(labels, dict): + continue + roles = {key.removeprefix("node-role.kubernetes.io/") for key in labels + if key.startswith("node-role.kubernetes.io/")} + status = node.get("status") or {} + conditions = status.get("conditions", []) if isinstance(status, dict) else [] + ready = isinstance(conditions, list) and any( + isinstance(c, dict) and c.get("type") == "Ready" and c.get("status") == "True" + for c in conditions) + spec = node.get("spec") or {} + if (ready and isinstance(spec, dict) and not spec.get("unschedulable") + and labels.get("kubernetes.io/arch") == "amd64" + and "worker" in roles and not roles.intersection({"master", "control-plane"}) + and _matches_selector(labels, placement)): + eligible.append(metadata.get("name")) + if not eligible or not set(eligible).intersection(installed_nodes): + raise PreflightError("KataWorkersNotReady") + if api_readiness.get("sandboxes.agents.x-k8s.io") is not True: + raise PreflightError("AgentSandboxUnavailable") + if api_readiness.get("externalsecrets.external-secrets.io") is not True: + raise PreflightError("SecretServicesUnavailable") + if api_readiness.get("adminnetworkpolicies.policy.networking.k8s.io") is not True: + raise PreflightError("NetworkPolicyRejected") + if api_readiness.get("admissionregistration.k8s.io/v1") is not True: + raise PreflightError("AdmissionUnavailable") + return tuple(sorted(eligible)) diff --git a/charts/openshell-rhdh/files/profile-catalog.json b/charts/openshell-rhdh/files/profile-catalog.json index 2ca481760..dcb7aef9d 100644 --- a/charts/openshell-rhdh/files/profile-catalog.json +++ b/charts/openshell-rhdh/files/profile-catalog.json @@ -28,6 +28,7 @@ { "enabled": true, "harnessRequired": false, + "image": "quay.io/aipcc/base-images/agentic/openclaw@sha256:896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", "name": "notebook", "type": "openclaw", "uiRoute": true @@ -35,6 +36,7 @@ { "enabled": false, "harnessRequired": false, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", "name": "cuda-sandbox", "type": "nemoclaw", "uiRoute": false @@ -42,6 +44,7 @@ { "enabled": false, "harnessRequired": false, + "image": "base", "name": "toolbox", "type": "generic", "uiRoute": false @@ -51,6 +54,9 @@ ] }, "custom-inference": { + "dedicatedEndpointSecrets": [ + "inference" + ], "description": "Agent workspace backed by a custom OpenAI-compatible endpoint (vLLM, Ollama, ...)", "harnessRequired": false, "secrets": { @@ -103,7 +109,10 @@ "sandboxes": [ { "enabled": true, + "endpointInput": "inference.url", "harnessRequired": false, + "image": "quay.io/aipcc/base-images/agentic/openclaw@sha256:896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", + "modelInput": "inference.model", "name": "notebook", "type": "openclaw", "uiRoute": true @@ -168,6 +177,8 @@ { "enabled": true, "harnessRequired": false, + "image": "quay.io/aipcc/base-images/agentic/openclaw@sha256:896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", + "modelInput": "inference.model", "name": "notebook", "type": "openclaw", "uiRoute": true @@ -175,6 +186,7 @@ { "enabled": false, "harnessRequired": false, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", "name": "cuda-sandbox", "type": "nemoclaw", "uiRoute": false @@ -182,6 +194,7 @@ { "enabled": false, "harnessRequired": false, + "image": "base", "name": "toolbox", "type": "generic", "uiRoute": false @@ -196,6 +209,8 @@ { "enabled": true, "harnessRequired": false, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", + "modelInput": "inference.cuda_model", "name": "cuda-sandbox", "type": "nemoclaw", "uiRoute": true @@ -203,6 +218,7 @@ { "enabled": false, "harnessRequired": false, + "image": "base", "name": "toolbox", "type": "generic", "uiRoute": false @@ -286,6 +302,7 @@ { "enabled": true, "harnessRequired": true, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", "name": "assistant", "type": "nemoclaw", "uiRoute": true diff --git a/charts/openshell-rhdh/files/saw_keycloak_admin.py b/charts/openshell-rhdh/files/saw_keycloak_admin.py new file mode 100644 index 000000000..90c1f4bd1 --- /dev/null +++ b/charts/openshell-rhdh/files/saw_keycloak_admin.py @@ -0,0 +1,36 @@ +"""Shared bootstrap of the built-in Keycloak Admin REST API audience scope. + +Canonical source here; ``make catalog`` packages the same module for RHDH. +The caller supplies its existing administrator-authenticated API transport. +This helper neither requests credentials nor grants administrative roles. +""" +from urllib.parse import quote + +ADMIN_API_SCOPE = "kc-admin-api" + + +def ensure_admin_api_scope(call, client_uuid): + """Assign the existing built-in scope without replacing other defaults. + + Keycloak 26.6.8 requires both this scope and its mapped Admin API audience. + Older servers do not have the scope: leave their configuration unchanged. + Lookup/authorization failures propagate; they are never treated as absence. + ``call`` accepts (method, realm-relative path, optional JSON body). + """ + scopes = call("GET", "/client-scopes") + if not isinstance(scopes, list): + raise RuntimeError("invalid Keycloak client scope response") + matching = [scope for scope in scopes if scope.get("name") == ADMIN_API_SCOPE] + if not matching: + return False + if (len(matching) != 1 or matching[0].get("protocol") != "openid-connect" + or not isinstance(matching[0].get("id"), str) or not matching[0]["id"]): + raise RuntimeError("invalid Keycloak Admin API scope") + scope_id = matching[0]["id"] + path = "/clients/" + quote(client_uuid, safe="") + "/default-client-scopes" + assigned = call("GET", path) + if not isinstance(assigned, list): + raise RuntimeError("invalid Keycloak default client scope response") + if not any(scope.get("id") == scope_id for scope in assigned): + call("PUT", path + "/" + quote(scope_id, safe="")) + return True diff --git a/charts/openshell-rhdh/files/set-workspace-state.yaml b/charts/openshell-rhdh/files/set-workspace-state.yaml new file mode 100644 index 000000000..06b69bc9f --- /dev/null +++ b/charts/openshell-rhdh/files/set-workspace-state.yaml @@ -0,0 +1,160 @@ +# Generated by scripts/saw-profile-catalog.py; do not edit. +apiVersion: scaffolder.backstage.io/v1beta3 +kind: Template +metadata: + name: set-saw-workspace-state + title: Stop or start my Container or Kata workspace + description: Pause or resume all sandboxes while retaining workspace files and audit history. + tags: [openshell, openclaw, nemoclaw, secure-agent-workspace] + labels: {saw.redhat.com/owners: 'true'} +spec: + owner: user:default/admin + type: agent-workspace + parameters: + - title: Workspace state + required: [operation] + properties: + operation: + type: string + enum: [stop, start] + steps: + - id: current + name: Read my current registration + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/workspace + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + - id: request + name: Submit the request + action: http:backstage:request + input: + method: POST + path: /proxy/saw-requests + headers: {Content-Type: application/json} + body: + apiVersion: v1 + kind: Secret + metadata: + generateName: saw-req- + labels: {saw.redhat.com/request: 'true'} + type: Opaque + stringData: {action: create, token: '${{ secrets.backstageToken }}', runtimeAction: '${{ parameters.operation + }}', registration: '${{ steps.current.output.body.registration }}', expectedGeneration: 'g-${{ + steps.current.output.body.generation }}'} + - id: run + name: Start the pipeline + action: http:backstage:request + input: + method: POST + path: /proxy/saw-pipelineruns + headers: {Content-Type: application/json} + body: + apiVersion: tekton.dev/v1 + kind: PipelineRun + metadata: + generateName: saw-create- + labels: {saw.redhat.com/request: 'true', backstage.io/kubernetes-id: 'saw-${{ user.ref | parseEntityRef + | pick(''name'') }}'} + spec: + pipelineRef: {name: saw-workspace-create} + timeouts: {pipeline: 1h30m} + taskRunTemplate: {serviceAccountName: __PROVISIONER_SA__} + params: + - {name: request, value: '${{ steps.request.output.body.metadata.name }}'} + - id: task-register + name: Verify the request, store the keys, register the workspace + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/run/${{ steps.run.output.body.metadata.name }}?task=register&wait=20 + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + continueOnBadResponse: true + each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + - id: task-argo-cd-apps + name: Argo CD creates the workspace's applications + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/run/${{ steps.run.output.body.metadata.name }}?task=argo-cd-apps&wait=20 + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + continueOnBadResponse: true + each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] + - id: task-vm + name: Provision the workspace runtime + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/run/${{ steps.run.output.body.metadata.name }}?task=vm&wait=20 + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + continueOnBadResponse: true + each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] + - id: task-vm-running + name: Start the workspace runtime + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/run/${{ steps.run.output.body.metadata.name }}?task=vm-running&wait=20 + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + continueOnBadResponse: true + each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51] + - id: task-sandboxes + name: Wait for OpenShell sandboxes and workspace access + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/run/${{ steps.run.output.body.metadata.name }}?task=sandboxes&wait=20 + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + continueOnBadResponse: true + each: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96] + - id: pipeline + name: Pipeline result + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/run/${{ steps.run.output.body.metadata.name }} + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + - id: pipeline-log + name: Pipeline log + action: debug:log + input: {message: '${{ steps[''pipeline''].output.body.text }}'} + - id: pipeline-check + name: Check the pipeline + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/run/${{ steps.run.output.body.metadata.name }}?assert=1 + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + - id: status + name: Workspace status + action: http:backstage:request + input: + method: GET + path: /proxy/saw-status/workspace + headers: {X-Saw-Token: '${{ secrets.backstageToken }}'} + timeout: 90000 + - id: status-log + name: Workspace status report + action: debug:log + input: {message: '${{ steps[''status''].output.body.text }}'} + output: + links: + - {title: Your workspace in the catalog, entityRef: 'component:default/saw-${{ user.ref | parseEntityRef + | pick(''name'') }}'} + text: + - {title: Workspace status, content: '${{ steps.status.output.body.text }}'} diff --git a/charts/openshell-rhdh/files/wait-oidc.py b/charts/openshell-rhdh/files/wait-oidc.py new file mode 100644 index 000000000..ce708b88a --- /dev/null +++ b/charts/openshell-rhdh/files/wait-oidc.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Gate RHDH startup on verified public OIDC discovery; never handles secrets.""" +import json +import ssl +import sys +import time +import urllib.parse +import urllib.request + + +def discover(issuer, context): + if urllib.parse.urlsplit(issuer).scheme != "https": + raise ValueError("OIDC issuer must use HTTPS") + request = urllib.request.Request( + issuer + "/.well-known/openid-configuration", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(request, context=context, timeout=10) as response: + if response.status != 200: + raise ValueError("OIDC discovery did not return HTTP 200") + if urllib.parse.urlsplit(response.geturl()).scheme != "https": + raise ValueError("OIDC discovery redirected away from HTTPS") + data = response.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + raise ValueError("OIDC discovery exceeds size limit") + metadata = json.loads(data) + if not isinstance(metadata, dict) or metadata.get("issuer") != issuer: + raise ValueError("OIDC discovery issuer mismatch") + for name in ("authorization_endpoint", "token_endpoint", "userinfo_endpoint", "jwks_uri"): + if urllib.parse.urlsplit(metadata.get(name, "")).scheme != "https": + raise ValueError("OIDC discovery lacks a secure endpoint") + + +def wait(issuer, cafile, timeout=600): + context = ssl.create_default_context(cafile=cafile) + deadline = time.monotonic() + timeout + while True: + try: + discover(issuer, context) + print("OIDC discovery ready; issuer and TLS verified", flush=True) + return 0 + except (OSError, ValueError, TypeError) as error: + # Never log response bodies, redirect URLs or credential material. + print("Waiting for OIDC discovery: " + type(error).__name__, flush=True) + remaining = deadline - time.monotonic() + if remaining <= 0: + print("OIDC discovery deadline exceeded; RHDH startup blocked", flush=True) + return 1 + time.sleep(min(5, remaining)) + + +if __name__ == "__main__": + sys.exit(wait(sys.argv[1], sys.argv[2])) diff --git a/charts/openshell-rhdh/files/workspace.py b/charts/openshell-rhdh/files/workspace.py new file mode 100644 index 000000000..bc86b4d01 --- /dev/null +++ b/charts/openshell-rhdh/files/workspace.py @@ -0,0 +1,410 @@ +"""Common workspace projection used by installation and portal adapters. + +Inputs are operator configuration, a BOM catalog and an immutable public +installation binding. Callers own authentication, credential storage, reads and +resource application. This module never discovers or changes cluster state. +""" +import hashlib +import json +import re +import urllib.parse +import urllib.error + + +class WorkspaceError(ValueError): + """Invalid workspace configuration or immutable publication.""" + + +SAFE_FAILURE_REASONS = frozenset({ + "ProfileRejected", "InvalidPlan", "PlacementRejected", "IdentityConflict", + "InvalidIntent", "StaleIntent", "ReconcilerStateRejected", "GatewayMismatch", + "ProviderProfileMismatch", "WorkspaceRejected", "ProviderRejected", + "SandboxRejected", "SandboxStateMismatch", "SandboxDrift", "CoreRejected", + "CoreFailure", +}) +SAFE_RPC_CODES = frozenset({ + "NOT_FOUND", "ALREADY_EXISTS", "UNAVAILABLE", "DEADLINE_EXCEEDED", "ABORTED", + "RESOURCE_EXHAUSTED", "PERMISSION_DENIED", "UNAUTHENTICATED", "INVALID_ARGUMENT", + "FAILED_PRECONDITION", "UNKNOWN", +}) + + +def enabled_catalog_sandboxes(profile): + for workspace in profile.get("workspaces", []): + if workspace.get("enabled"): + for sandbox in workspace.get("sandboxes", []): + if sandbox.get("enabled"): + yield workspace["name"], sandbox + + +def sandbox_targets(user, registration, profile, *, model_lifecycle=False): + """Expand the shared BOM catalog into isolated Kubernetes attachments.""" + enabled = list(enabled_catalog_sandboxes(profile)) + targets = {} + for workspace, sb in enabled: + key = workspace + "/" + sb["name"] + suffix = hashlib.sha256((registration+"/"+key).encode()).hexdigest()[:16] + single = len(enabled)==1 + targets[key] = dict(workspace=workspace, sandbox=sb["name"], ui=bool(sb.get("uiRoute")), + image=sb.get("image", ""), base=user if single else user+"-"+suffix[:8], + relayID=registration if single else suffix, + subjectMap="saw-portal-relay-subject" if single else "saw-portal-relay-"+suffix) + if model_lifecycle and sb.get("modelInput"): + targets[key]["modelInput"] = sb["modelInput"] + return targets + + +def public_model_inputs(profile, secrets): + """Only catalog-declared, noncredential model selectors can enter public intent.""" + values = {} + confidential = {secrets.get(name, {}).get(field["key"]) + for name, entry in profile.get("secrets", {}).items() + for field in entry["fields"] if field["kind"] == "secret"} + custom_models = {sb.get("modelInput") for _, sb in enabled_catalog_sandboxes(profile) if sb.get("endpointInput")} + for key in {sb["modelInput"] for _, sb in enabled_catalog_sandboxes(profile) if sb.get("modelInput")}: + secret, field = key.split(".",1) + definition = next((f for f in profile["secrets"][secret]["fields"] + if f["key"] == field and f["kind"] == "text"), None) + if definition is None: + raise WorkspaceError("invalid public model selector") + value = secrets.get(secret, {}).get(field) + if value is None or value == "": + continue + if not isinstance(value, str) or value in confidential: + raise WorkspaceError("invalid public model selector") + # An official provider may publish a qualified model ID as this field's + # BOM default. Other qualified IDs require the custom endpoint contract. + qualified = key in custom_models or value == definition.get("default") + spelling = r"[A-Za-z0-9][A-Za-z0-9._/-]{0,99}" if qualified else r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}" + if not re.fullmatch(spelling, value): + raise WorkspaceError("invalid public model selector") + values[key] = value + return values + + +def approved_endpoint(profile, secrets, boundary): + refs = {sb["endpointInput"] for _, sb in enabled_catalog_sandboxes(profile) if sb.get("endpointInput")} + if not refs: + return None + endpoint = boundary.get("inferenceBaseURL", "") + if not isinstance(endpoint, str) or not re.fullmatch( + r"https://[a-z0-9-]+(?:\.[a-z0-9-]+)+(?::443)?(?:/[A-Za-z0-9_-]+)*/v1", endpoint): + raise WorkspaceError("site has no approved custom inference endpoint") + if boundary.get("allowHosts") != [urllib.parse.urlsplit(endpoint).hostname]: + raise WorkspaceError("custom inference requires a dedicated broker host") + for ref in refs: + secret, key = ref.split(".", 1) + fields = profile["secrets"][secret]["fields"] + if not any(f["key"] == key and f["kind"] == "url" for f in fields): + raise WorkspaceError("endpoint input must be public URL metadata") + if secrets.get(secret, {}).get(key, "").rstrip("/") != endpoint: + raise WorkspaceError("endpoint differs from the site-approved custom inference URL") + return endpoint + + +def enrolled_site(site, ws, read): + """Only the trusted bootstrap publishes bindings; images, code and endpoints always come from Git.""" + a = site.get("automaticEnrollment") + if not a or not read: + return site + reg, install = ws["registration"], ws["installationID"] + ns = "saw-ctl-" + reg + def publication(name): + obj = read(f"/api/v1/namespaces/{ns}/configmaps/{name}") + if not obj or not obj.get("data"): + return {} + meta=obj["metadata"] + if obj.get("immutable") is not True or meta.get("deletionTimestamp") or meta.get("labels",{}).get("saw.redhat.com/runtime-owner")!=install: + raise WorkspaceError("invalid registration publication") + return obj["data"] + fresh = {k:v for k,v in site.items() if k!="enrollment"} + data=publication("saw-portal-enrollment") + if a.get("enabled") is False: + if data: + raise WorkspaceError("cannot disable enrollment of a bound workspace; use desiredState") + return fresh + for key, kind in (("reference", dict), ("relay", dict), ("revision", str), + ("image", str), ("buildNamespace", str)): + if not isinstance(a.get(key), kind) or not a[key]: + raise WorkspaceError(f"container site for {ws['name']} requires automaticEnrollment.{key}") + if not data: + return fresh + bound=json.loads(data["binding.json"]) + if set(bound)!={"registration","intent","approved","gatewayCA"} or bound["registration"]!=reg or set(bound["approved"])!=set(a["reference"]): + raise WorkspaceError("invalid published enrollment") + intent=bound["intent"] + if intent["bindings"]["ownerID"]!=install or intent["bindings"]["namespace"]!="saw-"+ws["name"] or intent["bindings"]["ownerSubject"]!=ws["ownerSubject"] or intent["profileRef"]["sha256"]!=site.get("selectedDigest",a.get("profileSHA256")): + raise WorkspaceError("published intent does not belong to this workspace") + subjects = {target["subjectMap"]:publication(target["subjectMap"]).get("subject", "") + for target in site.get("targets",{}).values() if target["ui"]} + fresh["subjects"] = subjects + sub=subjects.get("saw-portal-relay-subject","") if site.get("targets") else publication("saw-portal-relay-subject").get("subject","") + if sub and (not re.fullmatch(r"[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}",sub) or sub==ws["ownerSubject"]): + raise WorkspaceError("invalid relay subject") + relay={**a["relay"],"gatewayCA":bound["gatewayCA"],"subject":sub,"workspaceID":""} + fresh["enrollment"]={**{k:a[k] for k in ("revision","image","buildNamespace")},"registration":reg,"intent":intent,"approved":bound["approved"],"relay":relay} + return fresh + + +def previous_child(application, appset, owner): + """Read only the public values of the child controlled by this ApplicationSet.""" + if not application: + return None + meta = application["metadata"] + if (meta.get("deletionTimestamp") or meta.get("labels", {}).get("openshell.pattern/owner") != owner["name"] + or not any(ref.get("apiVersion") == "argoproj.io/v1alpha1" and ref.get("kind") == "ApplicationSet" + and ref.get("name") == appset and ref.get("controller") is True for ref in meta.get("ownerReferences", []))): + raise WorkspaceError("cannot verify existing workspace Application ownership") + source = application["spec"]["source"] + if source.get("path") != "charts/saw-users": + raise WorkspaceError("unexpected workspace Application source") + values = json.loads(source["helm"]["values"]) + users = values["users"] + if len(users) != 1 or any(users[0].get(k) != owner.get(k) for k in ("name", "installationID", "registration", "ownerSubject")): + raise WorkspaceError("existing Application belongs to a different registration") + if not isinstance(values, dict) or values["global"]["targetRevision"] != source["targetRevision"]: + raise WorkspaceError("existing Application revision does not match its values") + return values + + +def validate_runtime_placement(mode, site): + runtime = site.get("kataRuntimeClass") + if mode == "kata": + if (not isinstance(runtime, dict) or set(runtime) != {"name", "uid", "sha256"} + or not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", str(runtime["name"])) + or not re.fullmatch(r"[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}", str(runtime["uid"])) + or not re.fullmatch(r"[a-f0-9]{64}", str(runtime["sha256"]))): + raise WorkspaceError("kata requires a reviewed site RuntimeClass") + elif runtime is not None: + raise WorkspaceError("container registration cannot use a Kata approval") + + +def project_workspace(workspace, defaults, site, catalog, read=None, previous=None): + try: + values = _project_workspace(workspace, defaults, site, catalog, read) + name = workspace["name"] + projected = values.get("containerSites", {}).get(name, {}) + if previous and projected and site.get("automaticEnrollment"): + subjects = projected.get("subjects", {}) + incomplete = not projected.get("enrollment") or any( + target.get("ui") and not subjects.get(target["subjectMap"]) for target in projected["targets"].values()) + if incomplete: + retained = previous(workspace) + if retained: + old = retained["containerSites"][name] + if (old.get("enrollment") and not projected.get("enrollment") or + any(sub and not subjects.get(key) for key, sub in old.get("subjects", {}).items())): + # Returning the exact existing parameters holds only this child; + # omitting it could tell the static ApplicationSet to prune it. + return retained + return values + except WorkspaceError: + raise + except (KeyError, TypeError, ValueError, AttributeError) as exc: + # Inputs include operator configuration and immutable API publications. + # Do not include arbitrary values (especially JSON bodies) in diagnostics. + detail = f"missing {exc.args[0]}" if isinstance(exc, KeyError) and exc.args[0] in { + "subject", "registration", "installationID", "reference", "allowHosts", "bindings", + "binding.json", "profileRef", "sha256", "initialModelInputs"} else "invalid site or publication" + raise WorkspaceError(f"workspace {workspace.get('name', '?')}: {detail}") from None + + +def _project_workspace(workspace, defaults, site, catalog, read=None): + """Project an authenticated registration into common saw-users chart values. + + Entry points resolve the owner, choose a registration and load any public + enrollment objects. They must not manufacture a successful enrollment. + Neither RHDH nor a particular Kubernetes client is required here. + """ + ws = {**workspace, "values": {**workspace.get("values", {})}} + mode = ws["values"].get("deployment", {}).get("mode", "vm") + if mode not in {"vm", "container", "kata"}: + raise WorkspaceError("unsupported workspace deployment mode") + ws["values"]["deployment"] = {"mode": mode} + # This map belongs to the operator's installation configuration, never the + # registration or an owner request. Pin only code in the configured repository. + revisions = defaults.get("workspaceRevisions", {}) + if not isinstance(revisions, dict) or any( + not isinstance(name, str) or not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,17}[a-z0-9])?", name) + or not isinstance(revision, str) or not re.fullmatch(r"[a-f0-9]{40}", revision) + for name, revision in revisions.items()): + raise WorkspaceError("workspaceRevisions requires owner names and full Git commit SHAs") + values = {**{k:v for k,v in defaults.items() if k != "workspaceRevisions"}, "users": [ws]} + if ws.get("name") in revisions: + values["global"] = {**defaults.get("global", {}), "targetRevision": revisions[ws["name"]]} + if mode == "vm": + return values + if not isinstance(site, dict) or not site: + raise WorkspaceError(f"container site is missing for {ws['name']}") + if ws.get("ownerSubject") != site.get("subject"): + raise WorkspaceError("container site must bind the resolved owner subject") + validate_runtime_placement(mode, site) + automatic = site.get("automaticEnrollment", {}) + selected, *extra = ws.get("profiles", []) or [None] + foundation = automatic.get("enabled") is False + if extra or selected not in catalog or not foundation and selected not in automatic.get("approvedProfiles", {}): + raise WorkspaceError("container requires one approved BOM profile") + if not ws.get("registration") or not ws.get("installationID"): + raise WorkspaceError("container requires a registration and installation identity") + overrides = site.get("profileImageOverrides", {}).get(selected, {}) + if overrides: + ws["imageOverrides"] = overrides + managed = ws.get("lifecycleVersion") == 1 + site = {**site, + "targets": sandbox_targets(ws["name"], ws["registration"], catalog[selected], + model_lifecycle=managed), + "selectedDigest": automatic.get("approvedProfiles", {}).get(selected, ""), + "modelInputs": ws.get("modelInputs", {})} + for target in site["targets"].values(): + target["image"] = overrides.get(target["sandbox"], target["image"]) + boundary = site.get("profileBoundaries", {}).get(selected) + if boundary: + site = {**site, "providerAllowHosts": boundary["allowHosts"], + "automaticEnrollment": {**automatic, "reference": boundary["reference"]}} + if ws.get("customInferenceURL"): + if not boundary or boundary.get("inferenceBaseURL") != ws["customInferenceURL"]: + raise WorkspaceError("endpoint differs from the site-approved custom inference URL") + site["customInferenceURL"] = ws["customInferenceURL"] + if managed: + site.update(lifecycleVersion=1, modelInputs=ws["initialModelInputs"], + requestedModels=ws.get("modelInputs", {}), desiredState=ws["desiredState"], + restartGeneration=ws["restartGeneration"]) + values["containerSites"] = {ws["name"]: enrolled_site(site, ws, read)} + return values + + +# Preserve the existing lifecycle API stage IDs across entry points. +STAGE_IDS = ("registered", "apps", "vm", "running", "ready") + + +def sandbox_ui_links(user, profiles, catalog, domain): + out = [] + for profile in profiles: + for workspace, sb in enabled_catalog_sandboxes(catalog.get(profile, {})): + if sb.get("uiRoute"): + out.append((f"{sb['name']} UI ({workspace})", + f"https://{user}-{workspace}-{sb['name']}-ui.apps.{domain}")) + return out + + + +def container_status(read_resource, entry, site, argo_ns, domain, ping, catalog=None): + # Reuse the VM pipeline's wire stage IDs so old RHDH tasks can finish. + # Only observations and their user-facing names differ. + user = entry["name"] + ns = f"saw-{user}" + reg = entry["registration"] + rev = entry.get("intentGeneration") + gen = isinstance(rev, int) and not isinstance(rev, bool) and rev > 0 + app = read_resource(f"/apis/argoproj.io/v1alpha1/namespaces/{argo_ns}/applications/saw-{user}") or {} + meta = app.get("metadata", {}) + app_ok = meta.get("labels", {}).get("openshell.pattern/owner") == user and not meta.get("deletionTimestamp") + app_status = app.get("status", {}) + def read(name, field): + cm = read_resource(f"/api/v1/namespaces/{ns}/configmaps/{name}") or {} + meta = cm.get("metadata", {}) + labels = meta.get("labels", {}) + if (meta.get("deletionTimestamp") or not reg + or labels.get("saw.redhat.com/runtime-owner") != entry["installationID"] + or field == "status.json" and labels.get("saw.redhat.com/managed-by") != "kubernetes-runtime-controller"): + return {} + try: + value = json.loads(cm.get("data", {}).get(field, "{}")) + return value if isinstance(value, dict) else {} + except (ValueError, TypeError): + return {} + intent = read("saw-runtime-intent", "intent.json") if app_ok else {} + bound = bool(reg and site and gen and intent.get("generation") == rev + and intent.get("bindings", {}).get("ownerSubject") == site["subject"] + and intent.get("bindings", {}).get("namespace") == ns + and intent.get("bindings", {}).get("mode") == entry.get("values", {}).get("deployment", {}).get("mode", "container") + and intent.get("bindings", {}).get("ownerID") == entry["installationID"]) + desired = entry.get("desiredState", "Running") + bound = bound and intent.get("desiredState") == desired + st = read("saw-runtime-status", "status.json") if bound else {} + profile = (read_resource(f"/api/v1/namespaces/{ns}/configmaps/saw-bom-profiles") or {}) if bound else {} + files = profile.get("data", {}) + def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + ref = intent.get("profileRef", {}) + current = (bound and not profile.get("metadata", {}).get("deletionTimestamp") + and ref.get("name") == "saw-bom-profiles" and bool(ref.get("uid")) + and ref["uid"] == profile.get("metadata", {}).get("uid") and ref.get("sha256") == digest(files) + and type(st.get("observedGeneration")) is int and st["observedGeneration"] == rev + and st.get("inputRevision") == digest({"intent": intent, "files": files})) + running = current and st.get("phase") == "GenericReady" and intent.get("desiredState") == "Running" + links = sandbox_ui_links(user, entry.get("profiles", []), catalog, domain) if catalog and entry.get("desiredState") != "Stopped" else [] + profiles = entry.get("profiles", []) + declared = [sb for p in profiles for _, sb in enabled_catalog_sandboxes(catalog[p])] if ( + catalog and all(p in catalog for p in profiles)) else [] + headless = bool(declared) and all(sb.get("uiRoute") is False for sb in declared) + ready = bool(running and (headless or (links and domain)) and app_status.get("sync", {}).get("status") == "Synced" + and app_status.get("health", {}).get("status") == "Healthy" and all(ping(url) for _,url in links)) + observations = [(True, "Registration recorded"), (app_ok, "Argo application for this owner"), + (bound, "Intent for this registration and update"), + (running, "Profile sandboxes running"), (ready, "Workspace ready" if headless else "Owner chat routes available")] + stopped = bool(current and desired == "Stopped" and st.get("phase") == "Stopped") + if desired == "Stopped": + observations[-2:] = [(stopped, "OpenShell sandboxes stopped"), (stopped, "Storage retained")] + phase = "none" + steps = [] + waiting = False + for stage, (done, title) in zip(STAGE_IDS, observations): + state = "waiting" if waiting else "done" if done else "active" + if state == "done": + phase = stage + else: + waiting = True + steps.append(dict(id=stage, title=title, detail=title, state=state)) + failed = bool(current and st.get("phase") == "Blocked" or app_ok and app_status.get("operationState", {}).get("phase") in ("Error", "Failed")) + if failed: + reason = st.get("reason") + if isinstance(reason, str) and reason in SAFE_FAILURE_REASONS: + message = f"Container reconciliation failed ({reason})" + elif (reason == "CoreRpcRejected" and isinstance(st.get("rpcCode"), str) + and st["rpcCode"] in SAFE_RPC_CODES): + message = f"Container reconciliation failed (OpenShell {st['rpcCode']})" + else: + message = "Container reconciliation failed" + else: + message = next((s["title"] for s in steps if s["state"] == "active"), "Workspace ready") + if stopped and not failed: + message = "Workspace stopped" + phase = "stopped" + return dict(user=user, deploymentMode=entry.get("values", {}).get("deployment", {}).get("mode", "container"), phase="failed" if failed else phase, title=message, + registration=reg, generation=rev, observedGeneration=st.get("observedGeneration"), + desiredState=desired, modelInputs=entry.get("modelInputs", {}), + message=message, text=message, ready=ready and not failed, failed=failed, steps=steps, + links=[dict(title=title,url=url) for title,url in links] if ready and not failed else []) + + + +def owner_route_available(url, issuer, client, timeout=3): + """Require an anonymous challenge AND a registered callback at the trusted issuer.""" + origin = url.rstrip("/") + auth = issuer.rstrip("/") + "/protocol/openid-connect/auth" + if not issuer.startswith("https://") or not origin.startswith("https://"): + return False + location = origin + for hop in range(3): + try: + with client.open(location, timeout=timeout) as res: + return (location.split("?")[0] == auth and res.status == 200 + and b'id="kc-form-login"' in res.read(256 * 1024)) + except urllib.error.HTTPError as exc: + with exc: + target = urllib.parse.urljoin(location, exc.headers.get("Location", "")) + if exc.code not in (302, 303, 307): + return False + parts = urllib.parse.urlsplit(target) + if hop == 0 and target.split("?")[0] == origin + "/oauth2/start": + location = target + continue + params = urllib.parse.parse_qs(parts.query) + if (target.split("?")[0] != auth or params.get("client_id") != ["openshell-dashboard"] + or params.get("redirect_uri") != [origin + "/oauth2/callback"]): + return False + location = target + except (OSError, ValueError): + return False + return False diff --git a/charts/openshell-rhdh/templates/_helpers.tpl b/charts/openshell-rhdh/templates/_helpers.tpl index 8dececd94..985206768 100644 --- a/charts/openshell-rhdh/templates/_helpers.tpl +++ b/charts/openshell-rhdh/templates/_helpers.tpl @@ -23,11 +23,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- printf "https://%s/realms/%s" (include "openshell-rhdh.keycloakHost" .) .Values.keycloak.realm -}} {{- end -}} -{{/* The in-cluster RHDH service the pipeline fetches the JWKS from. */}} -{{- define "openshell-rhdh.internalUrl" -}} -{{- printf "http://backstage-%s.%s.svc:80" .Values.rhdh.name .Values.rhdh.namespace -}} -{{- end -}} - {{- /* The init container that writes the CA bundle (files/ca-bundle.py) to the saw-ca volume, and the volumes it needs; see tls in values.yaml. Each pod mounts saw-ca at /opt/saw-ca and reads /opt/saw-ca/ca-bundle.crt. */ -}} @@ -127,6 +122,7 @@ from the release is enough and keeps renders stable. {{- /* The governance engine decides each gateway's bindings (docs/apf.md). */ -}} {{- if .Values.global.governance -}}{{- $_ := set $g "governance" (deepCopy .Values.global.governance) -}}{{- end -}} {{- $_ := set $v "global" $g -}} +{{- $_ := set $v "containerSites" (.Values.portal.containerSites | default dict) -}} {{- $labels := deepCopy (index $v "namespaceLabels" | default dict) -}} {{- $_ := set $labels "saw.redhat.com/portal" "true" -}} {{- $_ := set $v "namespaceLabels" $labels -}} @@ -147,3 +143,122 @@ from the release is enough and keeps renders stable. {{- $found | default "rhdh.redhat.com/v1alpha5" -}} {{- end -}} {{- end -}} + +{{/* Operator-owned request policy; never derive it from a user request. */}} +{{- define "openshell-rhdh.deploymentModes" -}} +{{- $modes := .Values.portal.deploymentModes -}} +{{- if not (kindIs "slice" $modes) -}} +{{- fail "portal.deploymentModes must be a list containing vm" -}} +{{- end -}} +{{- $seen := dict -}} +{{- range $mode := $modes -}} +{{- if not (kindIs "string" $mode) -}} +{{- fail "portal.deploymentModes must contain only strings" -}} +{{- end -}} +{{- if not (has $mode (list "vm" "container" "kata")) -}} +{{- fail "portal.deploymentModes supports vm, container and kata" -}} +{{- end -}} +{{- if hasKey $seen $mode -}}{{- fail "portal.deploymentModes must not contain duplicates" -}}{{- end -}} +{{- $_ := set $seen $mode true -}} +{{- end -}} +{{- if not (hasKey $seen "vm") -}}{{- fail "portal.deploymentModes must include default vm" -}}{{- end -}} +{{- toJson $modes -}} +{{- end -}} + +{{/* Owner approvals can be rendered from values or read from a cluster Secret. */}} +{{- define "openshell-rhdh.containerSitesConfigured" -}} +{{- if and .Values.portal.containerSites .Values.portal.containerSitesSecretName -}} +{{- fail "set only one of portal.containerSites and portal.containerSitesSecretName" -}} +{{- end -}} +{{- if or .Values.portal.containerSites .Values.portal.containerSitesSecretName -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "openshell-rhdh.containerSitesEnv" -}} +- name: SAW_CONTAINER_SITES + {{- if .Values.portal.containerSitesSecretName }} + valueFrom: + secretKeyRef: + name: {{ .Values.portal.containerSitesSecretName | quote }} + key: container-sites.json + {{- else }} + value: {{ .Values.portal.containerSites | default dict | toJson | quote }} + {{- end }} +{{- end -}} + +{{/* Generated forms have vm as their safe default; project the administrator's +allowlist into both forms without enabling privileged form parameters. */}} +{{- define "openshell-rhdh.createTemplate" -}} +{{- $root := .root -}} +{{- $template := $root.Files.Get .path | fromYaml -}} +{{- $step := last $template.spec.parameters -}} +{{- $modes := include "openshell-rhdh.deploymentModes" $root | fromJsonArray -}} +{{- /* The form can advertise a site-wide union, never an owner authorization. +The server still verifies the authenticated owner's exact profile approval. */ -}} +{{- $approved := dict "container" (dict) "kata" (dict) -}} +{{- if $root.Values.portal.containerSitesSecretName -}} +{{- /* Secret contents are runtime-only. The pipeline performs owner-specific + checks; the form can show the enabled site modes and catalog profiles. */ -}} +{{- range $mode := list "container" "kata" -}} +{{- if has $mode $modes -}} +{{- range $id := $step.properties.profile.enum -}} +{{- $_ := set (index $approved $mode) $id true -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- else -}} +{{- range $site := $root.Values.portal.containerSites | default dict -}} +{{- if and (kindIs "map" $site) (regexMatch "^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$" (toString ($site.subject | default ""))) -}} +{{- $mode := "container" -}} +{{- if hasKey $site "kataRuntimeClass" -}} +{{- $class := $site.kataRuntimeClass -}} +{{- if not (and (kindIs "map" $class) (eq (len $class) 3) (regexMatch "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" (toString ($class.name | default ""))) (regexMatch "^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$" (toString ($class.uid | default ""))) (regexMatch "^[0-9a-f]{64}$" (toString ($class.sha256 | default "")))) -}} +{{- fail "Kata site requires a reviewed RuntimeClass name, UID and placement digest" -}}{{- end -}} +{{- $mode = "kata" -}} +{{- end -}} +{{- range $id, $digest := ($site.automaticEnrollment | default dict).approvedProfiles | default dict -}} +{{- if regexMatch "^[0-9a-f]{64}$" (toString $digest) -}}{{- $_ := set (index $approved $mode) $id true -}}{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- $profilesByMode := dict -}} +{{- range $mode := list "container" "kata" -}} +{{- $ids := list -}}{{- $names := list -}} +{{- range $i, $id := $step.properties.profile.enum -}} +{{- if hasKey (index $approved $mode) $id -}} +{{- $ids = append $ids $id -}}{{- $names = append $names (index $step.properties.profile.enumNames $i) -}} +{{- end -}} +{{- end -}} +{{- if not $ids -}}{{- $modes = without $modes $mode -}}{{- end -}} +{{- $_ := set $profilesByMode $mode (dict "enum" $ids "enumNames" $names) -}} +{{- end -}} +{{- $_ := set $step.properties.deploymentMode "enum" $modes -}} +{{- $labels := dict "vm" "Virtual machine" "container" "Container" "kata" "Kata" -}} +{{- $names := list -}} +{{- range $mode := $modes -}}{{- $names = append $names (index $labels $mode) -}}{{- end -}} +{{- $_ := set $step.properties.deploymentMode "enumNames" $names -}} +{{- /* Put the profile enum in each deployment branch rather than merging a +restricted enum with the broad generated field. Keep the generated profile +dependency and its credential fields shared by all deployment branches. */ -}} +{{- $profileField := deepCopy $step.properties.profile -}} +{{- $vmIDs := list -}}{{- $vmNames := list -}} +{{- range $i, $id := $profileField.enum -}} +{{- $vmIDs = append $vmIDs $id -}}{{- $vmNames = append $vmNames (index $profileField.enumNames $i) -}} +{{- end -}} +{{- $branches := list -}} +{{- range $mode := $modes -}} +{{- $field := deepCopy $profileField -}} +{{- if hasKey $profilesByMode $mode -}} +{{- $choices := index $profilesByMode $mode -}} +{{- $_ := set $field "enum" $choices.enum -}}{{- $_ := set $field "enumNames" $choices.enumNames -}} +{{- $_ := set $field "default" (first $choices.enum) -}} +{{- $_ := set $field "description" "Profiles configured for this deployment. Your administrator must also approve the selected profile for your account." -}} +{{- else -}} +{{- $_ := set $field "enum" $vmIDs -}}{{- $_ := set $field "enumNames" $vmNames -}} +{{- end -}} +{{- $branches = append $branches (dict "properties" (dict "deploymentMode" (dict "enum" (list $mode)) "profile" $field)) -}} +{{- end -}} +{{- $_ := unset $step.properties "profile" -}} +{{- $_ := set $step.dependencies "deploymentMode" (dict "oneOf" $branches) -}} +{{- include "openshell-rhdh.readinessTemplate" (dict "root" $root "template" $template) -}} +{{- end -}} diff --git a/charts/openshell-rhdh/templates/_readiness.tpl b/charts/openshell-rhdh/templates/_readiness.tpl new file mode 100644 index 000000000..386cd6dcf --- /dev/null +++ b/charts/openshell-rhdh/templates/_readiness.tpl @@ -0,0 +1,25 @@ +{{/* Match the duration returned by Tekton's Go API for GitOps comparison. */}} +{{- define "openshell-rhdh.minutesDuration" -}} +{{- if ge (int .) 60 -}}{{ printf "%dh%dm0s" (div (int .) 60) (mod (int .) 60) }}{{- else -}}{{ printf "%dm0s" (int .) }}{{- end -}} +{{- end -}} +{{/* Keep the UI's bounded status polling aligned with the Tekton wait. */}} +{{- define "openshell-rhdh.readinessTemplate" -}} +{{- $template := .template -}} +{{- $minutes := int .root.Values.portal.sandboxReadyTimeoutMinutes -}} +{{- $container := or (has "container" .root.Values.portal.deploymentModes) (has "kata" .root.Values.portal.deploymentModes) -}} +{{- $enrollmentTaskMinutes := add (max 10 $minutes) 2 -}} +{{- range $step := $template.spec.steps -}} +{{- if and $container (eq $step.id "task-vm") -}} +{{- $_ := set $step "each" (untilStep 1 (int (add (mul $enrollmentTaskMinutes 3) 1)) 1) -}} +{{- end -}} +{{- if and $container (eq $step.id "run") (eq $step.input.body.spec.pipelineRef.name "saw-workspace-create") -}} +{{- /* Register (5), apps (12), running (17), ready (+2), plus two minutes to schedule. */ -}} +{{- $_ := set $step.input.body.spec.timeouts "pipeline" (include "openshell-rhdh.minutesDuration" (max 90 (add 38 $enrollmentTaskMinutes $minutes))) -}} +{{- end -}} +{{- if eq $step.id "task-sandboxes" -}} +{{- $_ := set $step "each" (untilStep 1 (int (add (mul (add $minutes 2) 3) 1)) 1) -}} +{{- $_ := set $step "name" "Install OpenShell and wait for sandbox readiness" -}} +{{- end -}} +{{- end -}} +{{- toYaml $template | replace "__PROVISIONER_SA__" .root.Values.portal.provisionerServiceAccount -}} +{{- end -}} diff --git a/charts/openshell-rhdh/templates/applicationset.yaml b/charts/openshell-rhdh/templates/applicationset.yaml index fd267b1ea..ac49f2345 100644 --- a/charts/openshell-rhdh/templates/applicationset.yaml +++ b/charts/openshell-rhdh/templates/applicationset.yaml @@ -67,12 +67,20 @@ spec: namespace: {{ .Values.portal.namespace }} source: repoURL: {{ .Values.global.repoURL | quote }} - targetRevision: {{ .Values.global.targetRevision | quote }} + targetRevision: '{{ "{{" }} .revision {{ "}}" }}' path: charts/saw-users helm: releaseName: 'portal-ws-{{ "{{" }} .name {{ "}}" }}' values: '{{ "{{" }} .values {{ "}}" }}' + # Argo owns these two hook-state finalizers. Preserve them during sync; + # keep detecting changes to the resource-deletion finalizer and app spec. + ignoreDifferences: + - group: argoproj.io + kind: Application + jqPathExpressions: + - '.metadata.finalizers[]? | select(. == "pre-delete-finalizer.argocd.argoproj.io" or . == "pre-delete-finalizer.argocd.argoproj.io/cleanup")' syncPolicy: + syncOptions: [RespectIgnoreDifferences=true] automated: prune: true selfHeal: true diff --git a/charts/openshell-rhdh/templates/linked-runtime-reader.yaml b/charts/openshell-rhdh/templates/linked-runtime-reader.yaml new file mode 100644 index 000000000..60cf7483f --- /dev/null +++ b/charts/openshell-rhdh/templates/linked-runtime-reader.yaml @@ -0,0 +1,28 @@ +{{- range .Values.portal.linkedWorkspaces }} +{{- if .runtime }} +{{- $r := .runtime }} +{{- if ne $r.mode "container" }}{{ fail "linked runtime must be container" }}{{ end }} +{{- range $key := list "namespace" "intent" "status" }} +{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" (required $key (index $r $key))) }}{{ fail "invalid linked runtime reference" }}{{ end }} +{{- end }} +{{- $_ := required "installationID required" $r.installationID }} +{{- $_ := required "ownerSubject required" $r.ownerSubject }} +{{- $name := printf "saw-portal-status-%s" (printf "%s/%s/%s" $.Values.portal.namespace .owner .routeLabel | sha256sum | trunc 16) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $name }}, namespace: {{ $r.namespace }}} +rules: + - apiGroups: [""] + resources: [configmaps] + resourceNames: [{{ $r.intent | quote }}, {{ $r.status | quote }}] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $name }}, namespace: {{ $r.namespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }}} +subjects: + - {kind: ServiceAccount, name: saw-workspaces-generator, namespace: {{ $.Values.portal.namespace }}} +{{- end }} +{{- end }} diff --git a/charts/openshell-rhdh/templates/portal-ca.yaml b/charts/openshell-rhdh/templates/portal-ca.yaml index 90c91e9b3..90d7b653e 100644 --- a/charts/openshell-rhdh/templates/portal-ca.yaml +++ b/charts/openshell-rhdh/templates/portal-ca.yaml @@ -8,11 +8,15 @@ kind: ConfigMap metadata: name: saw-ca-sources namespace: {{ $ns }} + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" $ | nindent 4 }} data: ca-bundle.py: | {{ $.Files.Get "files/ca-bundle.py" | indent 4 }} + wait-oidc.py: | +{{ $.Files.Get "files/wait-oidc.py" | indent 4 }} extra-ca.crt: | {{ $.Values.tls.extraCaBundle | default "" | indent 4 }} --- @@ -23,6 +27,8 @@ kind: ConfigMap metadata: name: saw-trusted-ca namespace: {{ $ns }} + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" $ | nindent 4 }} config.openshift.io/inject-trusted-cabundle: "true" @@ -36,6 +42,8 @@ kind: Role metadata: name: saw-portal-ingress-ca namespace: openshift-config-managed + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} rules: @@ -49,6 +57,8 @@ kind: RoleBinding metadata: name: saw-portal-ingress-ca namespace: openshift-config-managed + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} roleRef: diff --git a/charts/openshell-rhdh/templates/portal-generator.yaml b/charts/openshell-rhdh/templates/portal-generator.yaml index 07349fb7a..f10c99330 100644 --- a/charts/openshell-rhdh/templates/portal-generator.yaml +++ b/charts/openshell-rhdh/templates/portal-generator.yaml @@ -31,7 +31,7 @@ spec: labels: app.kubernetes.io/name: saw-workspaces-generator annotations: - checksum/scripts: {{ .Files.Get "files/portal.py" | sha256sum | trunc 16 }} + checksum/scripts: {{ printf "%s%s%s" (.Files.Get "files/portal.py") (.Files.Get "files/workspace.py") (.Files.Get "files/preflight.py") | sha256sum | trunc 16 }} checksum/catalog: {{ .Files.Get "files/profile-catalog.json" | sha256sum | trunc 16 }} spec: serviceAccountName: saw-workspaces-generator @@ -51,6 +51,13 @@ spec: value: {{ $ns }} - name: PORT value: {{ $port | quote }} + {{- include "openshell-rhdh.containerSitesEnv" . | nindent 12 }} + - name: SAW_DEPLOYMENT_MODES + value: {{ include "openshell-rhdh.deploymentModes" . | quote }} + {{- with .Values.portal.linkedWorkspaces }} + - name: SAW_LINKED_WORKSPACES + value: {{ . | toJson | quote }} + {{- end }} - name: CATALOG_PATH value: /opt/saw/profile-catalog.json - name: GENERATOR_TOKEN_FILE @@ -61,7 +68,7 @@ spec: value: {{ include "openshell-rhdh.url" . }} # /status checks the caller's Backstage token against RHDH's keys. - name: RHDH_INTERNAL_URL - value: {{ include "openshell-rhdh.internalUrl" . }} + value: {{ include "openshell-rhdh.url" . }} - name: ARGO_NAMESPACE value: {{ include "openshell-rhdh.argoNamespace" . }} {{- if .Values.rhdh.rbac.enabled }} diff --git a/charts/openshell-rhdh/templates/portal-pipelines.yaml b/charts/openshell-rhdh/templates/portal-pipelines.yaml index 03e1e66f0..6988c0683 100644 --- a/charts/openshell-rhdh/templates/portal-pipelines.yaml +++ b/charts/openshell-rhdh/templates/portal-pipelines.yaml @@ -2,6 +2,16 @@ first task of each verifies the request and hands the user to the others as the result `user`; each later task is one stage the workspace goes through, so RHDH's Tekton tab draws a workspace's create and delete as a pipeline. */ -}} +{{- $readyMinutes := .Values.portal.sandboxReadyTimeoutMinutes -}} +{{- $sitesConfigured := eq (include "openshell-rhdh.containerSitesConfigured" .) "true" -}} +{{- if or (not (regexMatch "^[0-9]+$" (toString $readyMinutes))) (lt (int $readyMinutes) 1) (gt (int $readyMinutes) 60) -}} +{{- fail "portal.sandboxReadyTimeoutMinutes must be an integer from 1 to 60" -}} +{{- end -}} +{{- $taskTimeout := include "openshell-rhdh.minutesDuration" (add (int $readyMinutes) 2) -}} +{{- $intentMinutes := 10 -}} +{{- if or (has "container" .Values.portal.deploymentModes) (has "kata" .Values.portal.deploymentModes) -}} +{{- $intentMinutes = max 10 (int $readyMinutes) -}} +{{- end -}} apiVersion: tekton.dev/v1 kind: Task metadata: @@ -31,6 +41,10 @@ spec: - name: user description: The user the request was for. volumes: + {{- if $sitesConfigured }} + - name: owner-directory + secret: {secretName: saw-generator-keycloak, items: [{key: client_secret, path: client_secret}]} + {{- end }} - name: scripts configMap: name: saw-portal-scripts @@ -49,6 +63,9 @@ spec: # enabled on the Argo CD controller (off by default on a fresh install). computeResources: {} volumeMounts: + {{- if $sitesConfigured }} + - {name: owner-directory, mountPath: /etc/saw-owner-directory, readOnly: true} + {{- end }} - name: scripts mountPath: /opt/saw readOnly: true @@ -60,16 +77,36 @@ spec: value: /opt/saw-ca/ca-bundle.crt - name: NAMESPACE value: {{ .Values.portal.namespace }} + - name: SAW_CONTAINER_ENROLLMENT_SECONDS + value: {{ mul $intentMinutes 60 | quote }} - name: PIPELINE_RUN - value: $(context.pipelineRun.name) + valueFrom: + fieldRef: {fieldPath: "metadata.labels['tekton.dev/pipelineRun']"} - name: RESULT_PATH value: $(results.user.path) - name: CLUSTER_DOMAIN value: {{ .Values.global.clusterDomain | quote }} + {{- include "openshell-rhdh.containerSitesEnv" . | nindent 8 }} + - name: SAW_DEPLOYMENT_MODES + value: {{ include "openshell-rhdh.deploymentModes" . | quote }} - name: CATALOG_PATH value: /opt/saw/profile-catalog.json - name: RHDH_INTERNAL_URL - value: {{ include "openshell-rhdh.internalUrl" . }} + value: {{ include "openshell-rhdh.url" . }} + {{- if $sitesConfigured }} + - name: SAW_PORTAL_ADMIN_SUBJECTS + value: {{ .Values.portal.adminSubjects | default dict | toJson | quote }} + - name: KEYCLOAK_REALM + value: {{ .Values.keycloak.realm | quote }} + - name: KEYCLOAK_CLIENT_ID + value: {{ .Values.keycloak.clientId | quote }} + - name: KEYCLOAK_SECRET_FILE + value: /etc/saw-owner-directory/client_secret + - name: KEYCLOAK_CA_FILE + value: /opt/saw-ca/ca-bundle.crt + {{- end }} + - name: KEYCLOAK_URL + value: https://{{ include "openshell-rhdh.keycloakHost" . }} - name: ARGO_NAMESPACE value: {{ include "openshell-rhdh.argoNamespace" . }} - name: PORTAL_ADMINS @@ -104,9 +141,9 @@ spec: "create" (list (dict "name" "register" "action" "create" "timeout" "5m0s") (dict "name" "argo-cd-apps" "action" "wait-apps" "seconds" "600" "timeout" "12m0s") - (dict "name" "vm" "action" "wait-vm" "seconds" "600" "timeout" "12m0s") + (dict "name" "vm" "action" "wait-vm" "seconds" "600" "timeout" (include "openshell-rhdh.minutesDuration" (add $intentMinutes 2))) (dict "name" "vm-running" "action" "wait-running" "seconds" "900" "timeout" "17m0s") - (dict "name" "sandboxes" "action" "wait-ready" "seconds" "1800" "timeout" "32m0s")) + (dict "name" "sandboxes" "action" "wait-ready" "seconds" (printf "%d" (mul (int $readyMinutes) 60)) "timeout" $taskTimeout)) "delete" (list (dict "name" "unregister" "action" "delete" "timeout" "5m0s") (dict "name" "argo-cd-removes" "action" "wait-gone" "seconds" "1200" "timeout" "22m0s") diff --git a/charts/openshell-rhdh/templates/portal-rbac.yaml b/charts/openshell-rhdh/templates/portal-rbac.yaml index 5eff6ff1b..efcb46e86 100644 --- a/charts/openshell-rhdh/templates/portal-rbac.yaml +++ b/charts/openshell-rhdh/templates/portal-rbac.yaml @@ -166,6 +166,70 @@ subjects: - kind: ServiceAccount name: saw-workspaces-generator namespace: {{ $ns }} +{{- if or (has "container" .Values.portal.deploymentModes) (has "kata" .Values.portal.deploymentModes) }} +--- +# Read-only facts for the Kubernetes registration gate. This role checks node +# labels, runtime placement, admission resources, storage and service readiness. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: saw-portal-kata-preflight + labels: + {{- include "openshell-rhdh.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingadmissionpolicies", "validatingadmissionpolicybindings"] + verbs: ["get"] + - apiGroups: ["kataconfiguration.openshift.io"] + resources: ["kataconfigs"] + verbs: ["list"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["list"] + - apiGroups: ["external-secrets.io"] + resources: ["clustersecretstores"] + verbs: ["list"] + - apiGroups: ["config.openshift.io"] + resources: ["clusterversions"] + resourceNames: ["version"] + verbs: ["get"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: + - applications.argoproj.io + - applicationsets.argoproj.io + - kataconfigs.kataconfiguration.openshift.io + - sandboxes.agents.x-k8s.io + - externalsecrets.external-secrets.io + - adminnetworkpolicies.policy.networking.k8s.io + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: saw-portal-kata-preflight + labels: + {{- include "openshell-rhdh.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: saw-portal-kata-preflight +subjects: + - kind: ServiceAccount + name: {{ .Values.portal.provisionerServiceAccount }} + namespace: {{ $ns }} + {{- if ne .Values.portal.provisionerServiceAccount "saw-workspaces-generator" }} + - kind: ServiceAccount + name: saw-workspaces-generator + namespace: {{ $ns }} + {{- end }} +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/charts/openshell-rhdh/templates/portal-scripts.yaml b/charts/openshell-rhdh/templates/portal-scripts.yaml index 4912d56ff..7e7703e27 100644 --- a/charts/openshell-rhdh/templates/portal-scripts.yaml +++ b/charts/openshell-rhdh/templates/portal-scripts.yaml @@ -10,3 +10,8 @@ data: {{ .Files.Get "files/portal.py" | indent 4 }} profile-catalog.json: | {{ .Files.Get "files/profile-catalog.json" | indent 4 }} + + workspace.py: | +{{ .Files.Get "files/workspace.py" | indent 4 }} + preflight.py: | +{{ .Files.Get "files/preflight.py" | indent 4 }} diff --git a/charts/openshell-rhdh/templates/rhdh-app-config.yaml b/charts/openshell-rhdh/templates/rhdh-app-config.yaml index 591d693dc..970b50ae8 100644 --- a/charts/openshell-rhdh/templates/rhdh-app-config.yaml +++ b/charts/openshell-rhdh/templates/rhdh-app-config.yaml @@ -71,6 +71,12 @@ data: target: /opt/app-root/src/saw/delete-workspace.yaml rules: - allow: [Template] + {{- if or (has "container" (include "openshell-rhdh.deploymentModes" . | fromJsonArray)) (has "kata" (include "openshell-rhdh.deploymentModes" . | fromJsonArray)) }} + - type: file + target: /opt/app-root/src/saw/set-workspace-state.yaml + rules: + - allow: [Template] + {{- end }} {{- if .Values.portal.admins }} {{- range list "create-workspace-for-user.yaml" "delete-workspace-for-user.yaml" }} - type: file diff --git a/charts/openshell-rhdh/templates/rhdh-backstage.yaml b/charts/openshell-rhdh/templates/rhdh-backstage.yaml index faecfe8f4..af660e43d 100644 --- a/charts/openshell-rhdh/templates/rhdh-backstage.yaml +++ b/charts/openshell-rhdh/templates/rhdh-backstage.yaml @@ -51,17 +51,50 @@ spec: {{- if .Values.rhdh.host }} host: {{ .Values.rhdh.host }} {{- end }} - {{- if not .Values.tls.insecureSkipVerify }} - # The CA bundle NODE_EXTRA_CA_CERTS names, written before RHDH starts. + # RHDH mounts individual template files with subPath. A new Pod must load changes. deployment: patch: spec: + # Cold plugin installation can exceed Kubernetes' default ten-minute rollout deadline. + progressDeadlineSeconds: 1800 template: + metadata: + annotations: + saw.redhat.com/portal-templates: {{ include (print $.Template.BasePath "/rhdh-templates.yaml") . | sha256sum | quote }} + {{- if not .Values.tls.insecureSkipVerify }} + saw.redhat.com/oidc-startup-check: {{ .Files.Get "files/wait-oidc.py" | sha256sum | quote }} spec: initContainers: {{- include "openshell-rhdh.caInitContainer" . | nindent 14 }} + # Stock OIDC caches initial discovery, including a startup failure. + # Wait for the issuer before the backend initializes that provider. + - name: saw-oidc-discovery + image: {{ .Values.portal.image }} + command: + - python3 + - /opt/saw-ca-sources/wait-oidc.py + - {{ include "openshell-rhdh.issuer" . | quote }} + - /opt/saw-ca/ca-bundle.crt + volumeMounts: + - name: saw-ca + mountPath: /opt/saw-ca + readOnly: true + - name: saw-ca-sources + mountPath: /opt/saw-ca-sources + readOnly: true + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { memory: 64Mi } + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: { drop: [ALL] } + seccompProfile: { type: RuntimeDefault } containers: - name: backstage-backend + # Preserve the operator's health endpoint and allow five minutes for backend initialization. + startupProbe: + failureThreshold: 15 volumeMounts: - name: saw-ca mountPath: /opt/saw-ca diff --git a/charts/openshell-rhdh/templates/rhdh-keycloak-client.yaml b/charts/openshell-rhdh/templates/rhdh-keycloak-client.yaml index f1a480d7b..1606b33d1 100644 --- a/charts/openshell-rhdh/templates/rhdh-keycloak-client.yaml +++ b/charts/openshell-rhdh/templates/rhdh-keycloak-client.yaml @@ -4,6 +4,8 @@ kind: ServiceAccount metadata: name: saw-rhdh-keycloak namespace: {{ .Values.rhdh.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} --- @@ -12,6 +14,8 @@ kind: Role metadata: name: saw-rhdh-keycloak-admin namespace: {{ .Values.keycloak.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} rules: @@ -25,6 +29,8 @@ kind: RoleBinding metadata: name: saw-rhdh-keycloak-admin namespace: {{ .Values.keycloak.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} roleRef: @@ -41,9 +47,13 @@ kind: ConfigMap metadata: name: saw-rhdh-keycloak-client namespace: {{ .Values.rhdh.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} data: + saw_keycloak_admin.py: | +{{ .Files.Get "files/saw_keycloak_admin.py" | indent 4 }} keycloak-client.py: | {{ .Files.Get "files/keycloak-client.py" | indent 4 }} --- @@ -55,9 +65,12 @@ metadata: labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} annotations: - argocd.argoproj.io/hook: PostSync + # ESO, CA inputs and scoped RBAC precede this hook. Consumers follow at wave 0. + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/hook: Sync argocd.argoproj.io/hook-delete-policy: BeforeHookCreation spec: + activeDeadlineSeconds: 900 backoffLimit: 10 template: metadata: diff --git a/charts/openshell-rhdh/templates/rhdh-oidc.yaml b/charts/openshell-rhdh/templates/rhdh-oidc.yaml index 50e9cf001..68d0c1559 100644 --- a/charts/openshell-rhdh/templates/rhdh-oidc.yaml +++ b/charts/openshell-rhdh/templates/rhdh-oidc.yaml @@ -4,6 +4,8 @@ kind: ExternalSecret metadata: name: rhdh-oidc namespace: {{ .Values.rhdh.namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-2" labels: {{- include "openshell-rhdh.labels" . | nindent 4 }} spec: diff --git a/charts/openshell-rhdh/templates/rhdh-templates.yaml b/charts/openshell-rhdh/templates/rhdh-templates.yaml index 9899aca72..700527fe6 100644 --- a/charts/openshell-rhdh/templates/rhdh-templates.yaml +++ b/charts/openshell-rhdh/templates/rhdh-templates.yaml @@ -9,13 +9,17 @@ metadata: {{- include "openshell-rhdh.labels" . | nindent 4 }} data: create-workspace.yaml: | -{{ .Files.Get "files/create-workspace.yaml" | replace "__PROVISIONER_SA__" .Values.portal.provisionerServiceAccount | indent 4 }} +{{ include "openshell-rhdh.createTemplate" (dict "root" . "path" "files/create-workspace.yaml") | indent 4 }} + {{- if or (has "container" (include "openshell-rhdh.deploymentModes" . | fromJsonArray)) (has "kata" (include "openshell-rhdh.deploymentModes" . | fromJsonArray)) }} + set-workspace-state.yaml: | +{{ include "openshell-rhdh.readinessTemplate" (dict "root" . "template" (.Files.Get "files/set-workspace-state.yaml" | fromYaml)) | indent 4 }} + {{- end }} delete-workspace.yaml: | {{ .Files.Get "files/delete-workspace.yaml" | replace "__PROVISIONER_SA__" .Values.portal.provisionerServiceAccount | indent 4 }} {{- if .Values.portal.admins }} # Administrators (portal.admins): a workspace for another user. create-workspace-for-user.yaml: | -{{ .Files.Get "files/create-workspace-for-user.yaml" | replace "__PROVISIONER_SA__" .Values.portal.provisionerServiceAccount | indent 4 }} +{{ include "openshell-rhdh.createTemplate" (dict "root" . "path" "files/create-workspace-for-user.yaml") | indent 4 }} delete-workspace-for-user.yaml: | {{ .Files.Get "files/delete-workspace-for-user.yaml" | replace "__PROVISIONER_SA__" .Values.portal.provisionerServiceAccount | indent 4 }} {{- end }} diff --git a/charts/openshell-rhdh/values.yaml b/charts/openshell-rhdh/values.yaml index 259c0c8a4..edf1d4f3e 100644 --- a/charts/openshell-rhdh/values.yaml +++ b/charts/openshell-rhdh/values.yaml @@ -96,10 +96,26 @@ rhdh: # The portal namespace: request Secrets, the Tekton pipelines, the workspace # registry (ConfigMaps saw-ws-) and the ApplicationSet plugin generator. portal: + # Owner-visible catalog links for independently GitOps-managed workspaces. + linkedWorkspaces: [] + # Administrator-owned request allowlist. Kata appears only for owners with a + # reviewed RuntimeClass and an approved profile. Existing workspace mode is immutable. + deploymentModes: [vm, kata] + containerSites: {} # Administrator-owned, keyed by login; empty denies all container requests. + # Use an existing Secret in portal.namespace for owner approvals that must + # stay out of Git. Its container-sites.json key has the same JSON shape as + # containerSites. When set, the form shows enabled modes and profiles; the + # provisioner still checks the exact owner and profile against the Secret. + containerSitesSecretName: "" namespace: saw-portal provisionerServiceAccount: saw-portal-provisioner # Python 3 with the standard library is all portal.py needs. image: registry.access.redhat.com/ubi9/python-311:latest + # Cold agent images on nested clusters may exceed the default. The portal + # still requires every expected UI route; this never marks a timeout Ready. + # Container registration also uses this budget (at least 10 minutes) to wait for its reviewed intent. + # Applies to newly created PipelineRuns, not a run already in progress. + sandboxReadyTimeoutMinutes: 30 # A portal workspace is deleted with its VM when it is removed. pruneOnRemove: true # delete also removes the user's keys from Vault. @@ -108,6 +124,8 @@ portal: # (the "... for a user" templates). Keycloak user names. Everyone else acts # only for themselves. admins: [] + # Immutable Keycloak subjects for administrators acting for container owners. + adminSubjects: {} generator: port: 4355 # Only RHDH (catalog) and Argo CD (ApplicationSet plugin) may reach the diff --git a/charts/openshell-saw-kubernetes/Chart.yaml b/charts/openshell-saw-kubernetes/Chart.yaml new file mode 100644 index 000000000..5782371f8 --- /dev/null +++ b/charts/openshell-saw-kubernetes/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: openshell-saw-kubernetes +description: Experimental SAW integration for the stock OpenShell Kubernetes driver +type: application +version: 0.1.0 +appVersion: "0.1.6-rhaiv.4" diff --git a/charts/openshell-saw-kubernetes/templates/NOTES.txt b/charts/openshell-saw-kubernetes/templates/NOTES.txt new file mode 100644 index 000000000..fd9185536 --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/NOTES.txt @@ -0,0 +1,7 @@ +Container gateway foundation using the stock OpenShell gateway and supervisor. + +Install native admission/network policy, retained identity, the credential proxy, +governance and Agent Sandbox through their declared owners. Workspace preparation, +reconciliation and browser access are composed by saw-users. + +See docs/container-installation.md and docs/container-workspaces.md. diff --git a/charts/openshell-saw-kubernetes/templates/_helpers.tpl b/charts/openshell-saw-kubernetes/templates/_helpers.tpl new file mode 100644 index 000000000..cccd7beed --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{- define "saw-kubernetes.name" -}} +{{- .Release.Name -}} +{{- end -}} +{{- define "saw-kubernetes.labels" -}} +app.kubernetes.io/name: {{ include "saw-kubernetes.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/part-of: secure-agent-workspace +{{- end -}} +{{- define "saw-kubernetes.validate" -}} +{{- $k := .Values.kubernetes -}} +{{- if not $k.experimental -}}{{- fail "Kubernetes integration is experimental; set kubernetes.experimental=true only for qualification" -}}{{- end -}} +{{- if not (has .Values.deployment.mode (list "container" "kata")) -}}{{- fail "unsupported Kubernetes workspace mode" -}}{{- end -}} +{{- if or (gt (len .Release.Name) 19) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" .Release.Name)) -}}{{- fail "release name must be a DNS label of at most 19 characters" -}}{{- end -}} +{{- $ns := .Release.Namespace -}} +{{- if or (gt (len $ns) 50) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $ns)) (eq $ns "default") (hasPrefix "openshift" $ns) (hasPrefix "kube-" $ns) -}}{{- fail "gateway requires a dedicated non-system namespace of at most 50 characters" -}}{{- end -}} +{{- if not (or (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1beta1/Sandbox") (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1beta1")) -}}{{- fail "install the pinned Agent Sandbox v1beta1 CRD/controller before this chart" -}}{{- end -}} +{{- if not (.Capabilities.APIVersions.Has "policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy") -}}{{- fail "OpenShift AdminNetworkPolicy is required for supervisor egress isolation" -}}{{- end -}} +{{- range $name, $image := dict "gateway" $k.gateway.image "supervisor" $k.supervisor.image "workload" $k.workload.image "runtime" $k.workload.runtimeImage -}} +{{- if not (regexMatch "^.+@sha256:[0-9a-f]{64}$" $image) -}}{{- fail (printf "%s image must be pinned by sha256 digest" $name) -}}{{- end -}} +{{- end -}} +{{- range $name, $secret := $k.identity -}} +{{- if or (not $secret) (gt (len $secret) 253) (not (regexMatch "^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" $secret)) -}}{{- fail (printf "kubernetes.identity.%s requires an existing retained Secret name" $name) -}}{{- end -}} +{{- end -}} +{{- if not $k.storage.className -}}{{- fail "kubernetes.storage.className must be explicit" -}}{{- end -}} +{{- if and (eq .Values.deployment.mode "container") $k.workload.runtimeClassName -}}{{- fail "container mode must use the default workload runtime" -}}{{- end -}} +{{- if and (eq .Values.deployment.mode "kata") (not (regexMatch "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" $k.workload.runtimeClassName)) -}}{{- fail "kata requires an approved workload RuntimeClass name" -}}{{- end -}} +{{- if not (regexMatch "^https://[^/?#@[:space:]]+(/[^?#[:space:]]*)?$" .Values.oidc.issuerUrl) -}}{{- fail "oidc.issuerUrl must be explicit HTTPS without userinfo, query or fragment" -}}{{- end -}} +{{- if not (or (regexMatch "^https://[^/?#@[:space:]]+/?$" $k.governance.endpoint) (regexMatch "^http://[a-z0-9-]+\\.[a-z0-9-]+\\.svc(\\.cluster\\.local)?:[0-9]+$" $k.governance.endpoint)) -}}{{- fail "governance endpoint must be HTTPS without userinfo or the explicitly selected in-cluster HTTP gRPC Service" -}}{{- end -}} +{{- if or (not (regexMatch "^https?://[^/?#@[:space:]]+:[0-9]+$" $k.proxy.url)) (not $k.proxy.caConfigMap) -}}{{- fail "configure the trusted credential proxy URL without userinfo and public CA ConfigMap" -}}{{- end -}} +{{- if or (not $k.network.reconcilerNamespace) (not $k.network.reconcilerLabels) -}}{{- fail "trusted reconciler namespace and exact Pod labels are required" -}}{{- end -}} +{{- if eq $k.network.reconcilerNamespace .Release.Namespace -}}{{- fail "trusted reconciler identity must be outside the runtime namespace" -}}{{- end -}} +{{- if not $k.network.gatewayEgress -}}{{- fail "explicit gateway API/DNS/OIDC/governance/proxy egress rules are required" -}}{{- end -}} +{{- range $rule := $k.network.gatewayEgress -}} +{{- if or (not $rule.to) (not $rule.ports) -}}{{- fail "gateway egress rules need nonempty destinations and ports" -}}{{- end -}} +{{- range $peer := $rule.to -}} +{{- if not $peer -}}{{- fail "gateway egress must not contain an empty peer" -}}{{- end -}} +{{- if hasKey $peer "namespaceSelector" -}} +{{- if and (empty $peer.namespaceSelector.matchLabels) (empty $peer.namespaceSelector.matchExpressions) -}}{{- fail "gateway egress must not select all namespaces" -}}{{- end -}} +{{- end -}} +{{- if and $peer.ipBlock (has $peer.ipBlock.cidr (list "0.0.0.0/0" "::/0")) (not $peer.ipBlock.except) -}}{{- fail "gateway egress must not allow all addresses" -}}{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/openshell-saw-kubernetes/templates/gateway-admission.yaml b/charts/openshell-saw-kubernetes/templates/gateway-admission.yaml new file mode 100644 index 000000000..987d43bf4 --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/gateway-admission.yaml @@ -0,0 +1,92 @@ +{{- $root := . }} +{{- range $resource := list "secrets" "services" "pods" }} +{{- $name := printf "%s-%s-driver" $root.Release.Namespace $resource }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ $name }} + annotations: {argocd.argoproj.io/sync-wave: "-12"} +spec: + failurePolicy: Fail + matchConstraints: + namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $root.Release.Namespace }}}} + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + resources: [{{ $resource }}] + operations: {{ if eq $resource "pods" }}[DELETE]{{ else }}[CREATE, UPDATE, DELETE]{{ end }} + matchConditions: + - name: stock-gateway + expression: request.userInfo.username == {{ printf "system:serviceaccount:%s:%s-gateway" $root.Release.Namespace $root.Release.Name | toJson }} +{{- if eq $resource "pods" }} + validations: + - expression: >- + (oldObject != null && + oldObject.spec.serviceAccountName == {{ printf "%s-sandbox" $root.Release.Name | toJson }} && + oldObject.metadata.ownerReferences.size() == 1 && oldObject.metadata.ownerReferences[0].kind == 'Sandbox' && + oldObject.metadata.ownerReferences[0].apiVersion in ['agents.x-k8s.io/v1beta1','agents.x-k8s.io/v1alpha1'] && + oldObject.metadata.labels['openshell.ai/boundary-role'] in ['workload','supervisor']) + message: The gateway may delete only Sandbox-owned runtime Pods. +{{- else }} + variables: + - name: obj + expression: "request.operation == 'DELETE' ? oldObject : object" + - name: id + expression: >- + has(variables.obj.metadata.labels) && 'openshell.ai/sandbox-id' in variables.obj.metadata.labels ? + variables.obj.metadata.labels['openshell.ai/sandbox-id'] : '' + validations: + - expression: >- + variables.id.matches('^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$') && + has(variables.obj.metadata.ownerReferences) && variables.obj.metadata.ownerReferences.size() == 1 + message: Driver resources require a sandbox identity and owner. +{{- if eq $resource "secrets" }} + - expression: >- + variables.obj.type == 'Opaque' && has(variables.obj.immutable) && variables.obj.immutable && + variables.obj.metadata.name.matches('^os-(sandbox|supervisor)-' + variables.id + '-[a-f0-9]{12}$') && + variables.obj.metadata.ownerReferences[0].kind == 'Pod' && + variables.obj.metadata.ownerReferences[0].apiVersion == 'v1' + message: The driver may manage only its immutable Pod-owned bootstrap Secrets. +{{- else }} + - expression: >- + ( + variables.obj.metadata.name == 'os-boundary-' + variables.id && + variables.obj.metadata.ownerReferences[0].kind == 'Sandbox' && + variables.obj.metadata.ownerReferences[0].apiVersion in ['agents.x-k8s.io/v1beta1','agents.x-k8s.io/v1alpha1'] && + variables.obj.spec.type == 'ClusterIP' && !has(variables.obj.spec.externalIPs) && + !has(variables.obj.spec.externalName) && variables.obj.spec.selector.size() == 2 && + variables.obj.spec.selector['openshell.ai/boundary-role'] == 'workload' && + variables.obj.spec.selector['openshell.ai/boundary-pair'] == variables.id && + variables.obj.spec.ports.size() == 1 && variables.obj.spec.ports[0].name == 'boundary' && + variables.obj.spec.ports[0].protocol == 'TCP' && variables.obj.spec.ports[0].port == 5500 && + variables.obj.spec.ports[0].targetPort == 5500 + ) || + ( + variables.obj.metadata.name == 'os-supervisor-' + variables.id && + variables.obj.metadata.ownerReferences[0].kind == 'Sandbox' && + variables.obj.metadata.ownerReferences[0].apiVersion in ['agents.x-k8s.io/v1beta1','agents.x-k8s.io/v1alpha1'] && + variables.obj.spec.type == 'ClusterIP' && !has(variables.obj.spec.externalIPs) && + !has(variables.obj.spec.externalName) && variables.obj.spec.selector.size() >= 2 && + variables.obj.spec.selector['openshell.ai/boundary-role'] == 'supervisor' && + (('openshell.ai/boundary-pair' in variables.obj.spec.selector && + variables.obj.spec.selector['openshell.ai/boundary-pair'] == variables.id) || + ('openshell.ai/sandbox-id' in variables.obj.spec.selector && + variables.obj.spec.selector['openshell.ai/sandbox-id'] == variables.id)) && + variables.obj.spec.ports.size() == 1 && variables.obj.spec.ports[0].protocol == 'TCP' && + variables.obj.spec.ports[0].port == 5500 && variables.obj.spec.ports[0].targetPort == 5500 && + has(variables.obj.spec.publishNotReadyAddresses) && variables.obj.spec.publishNotReadyAddresses + ) + message: The driver may create only internal per-sandbox boundary and supervisor Services. +{{- end }} +{{- end }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ $name }} + annotations: {argocd.argoproj.io/sync-wave: "-11"} +spec: + policyName: {{ $name }} + validationActions: [Deny] +{{- end }} diff --git a/charts/openshell-saw-kubernetes/templates/gateway-config.yaml b/charts/openshell-saw-kubernetes/templates/gateway-config.yaml new file mode 100644 index 000000000..b0143a11c --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/gateway-config.yaml @@ -0,0 +1,87 @@ +{{/* Configuration fields verified against stock ODH OpenShell 305c8b3. */}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-gateway-config + labels: + {{- include "saw-kubernetes.labels" . | nindent 4 }} +data: + gateway.toml: | + [openshell] + version = 2 + [openshell.gateway] + name = {{ .Release.Name | quote }} + bind_address = "0.0.0.0:17670" + health_bind_address = "0.0.0.0:8081" + compute_driver = "kubernetes" + policy_validation_failure_mode = "fail_closed" + enable_loopback_service_http = false + enable_websocket_tunnel = true + provider_profile_sources = [{ type = "interceptor", name = "governance" }] + [openshell.gateway.auth] + allow_unauthenticated_users = false + [openshell.gateway.mtls_auth] + enabled = false + operator_enabled = true + [openshell.gateway.tls] + cert_path = "/etc/openshell-tls/server/tls.crt" + key_path = "/etc/openshell-tls/server/tls.key" + client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" + [openshell.gateway.gateway_jwt] + signing_key_path = "/etc/openshell-jwt/signing.pem" + public_key_path = "/etc/openshell-jwt/public.pem" + kid_path = "/etc/openshell-jwt/kid" + gateway_id = {{ .Release.Name | quote }} + ttl_secs = 3600 + [openshell.gateway.credential_storage] + key_encryption_key_env = "OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY" + [openshell.gateway.oidc] + issuer = {{ .Values.oidc.issuerUrl | quote }} + audience = {{ .Values.oidc.clientId | quote }} + roles_claim = {{ .Values.oidc.rolesClaim | quote }} + admin_role = {{ .Values.oidc.adminRole | quote }} + user_role = {{ .Values.oidc.userRole | quote }} + dangerously_allow_insecure_http = false + [openshell.drivers.kubernetes] + namespace = {{ .Release.Namespace | quote }} + gateway_id = {{ .Release.Name | quote }} + workspace_mode = "shared" + service_account_name = {{ printf "%s-sandbox" .Release.Name | quote }} + grpc_endpoint = {{ printf "https://%s-gateway.%s.svc:17670" .Release.Name .Release.Namespace | quote }} + default_image = {{ .Values.kubernetes.workload.image | quote }} + supervisor_image = {{ .Values.kubernetes.supervisor.image | quote }} + sandbox_runtime_image = {{ .Values.kubernetes.workload.runtimeImage | quote }} + # Driver projects only ca.crt: supervisors must trust the SERVER root. + client_tls_secret_name = {{ .Values.kubernetes.identity.serverTlsSecret | quote }} + default_runtime_class_name = {{ .Values.kubernetes.workload.runtimeClassName | quote }} + workspace_storage_class = {{ .Values.kubernetes.storage.className | quote }} + workspace_default_storage_size = {{ .Values.kubernetes.storage.workspaceSize | quote }} + allow_driver_config = true + sa_token_ttl_secs = 3600 + image_pull_policy = "IfNotPresent" + supervisor_image_pull_policy = "IfNotPresent" + sandbox_runtime_image_pull_policy = "IfNotPresent" + image_pull_secrets = {{ .Values.kubernetes.imagePullSecrets | toJson }} + https_proxy = {{ .Values.kubernetes.proxy.url | quote }} + # Stock SAW proxy accepts DNS:443 CONNECT targets, never literal IPs. + proxy_connect_by_hostname = true + proxy_ca_bundle = "/etc/openshell-tls/proxy-ca/ca.crt" + [openshell.drivers.kubernetes.resource_admission] + enabled = true + [openshell.drivers.kubernetes.resource_admission.required_labels] + "openshell.ai/sandbox-attachable" = "true" + "openshell.ai/sandbox-attachable-workspace" = "${workspace}" + [openshell.drivers.kubernetes.sandbox_runtime] + boundary_port = 5500 + [[openshell.gateway.interceptors]] + name = "governance" + grpc_endpoint = {{ .Values.kubernetes.governance.endpoint | quote }} + failure_policy = "fail_closed" + binding_policy = "allowlist" + timeout = "500ms" + allow_insecure_transport = {{ hasPrefix "http://" .Values.kubernetes.governance.endpoint }} + {{- range $rpc, $phases := dict "CreateSandbox" (list "modify_operation" "validate") "CreateProvider" (list "validate") "UpdateConfig" (list "validate") "SubmitPolicyAnalysis" (list "validate") "ImportProviderProfiles" (list "validate") "UpdateProviderProfiles" (list "validate") "DeleteProviderProfile" (list "validate") }} + [[openshell.gateway.interceptors.bindings]] + rpc = {{ printf "openshell.v1.OpenShell/%s" $rpc | quote }} + phases = {{ $phases | toJson }} + {{- end }} diff --git a/charts/openshell-saw-kubernetes/templates/gateway.yaml b/charts/openshell-saw-kubernetes/templates/gateway.yaml new file mode 100644 index 000000000..d56efaebb --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/gateway.yaml @@ -0,0 +1,131 @@ +{{- $k := .Values.kubernetes -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }}-gateway + labels: + {{- include "saw-kubernetes.labels" . | nindent 4 }} +spec: + replicas: 1 + serviceName: {{ .Release.Name }}-gateway-peer + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + openshell.pattern/role: gateway + template: + metadata: + labels: + {{- include "saw-kubernetes.labels" . | nindent 8 }} + openshell.pattern/role: gateway + annotations: + checksum/config: {{ include (print $.Template.BasePath "/gateway-config.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ .Release.Name }}-gateway + securityContext: + runAsNonRoot: true + seccompProfile: {type: RuntimeDefault} + {{- if $k.imagePullSecrets }} + imagePullSecrets: + {{- range $k.imagePullSecrets }} + - name: {{ . | quote }} + {{- end }} + {{- end }} + containers: + - name: gateway + image: {{ $k.gateway.image | quote }} + args: [--config, /etc/openshell/gateway.toml, --db-url, "sqlite:///var/openshell/gateway.db"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + env: + - name: OPENSHELL_REPLICA_ID + valueFrom: {fieldRef: {fieldPath: metadata.name}} + - name: OPENSHELL_POD_NAME + valueFrom: {fieldRef: {fieldPath: metadata.name}} + - name: OPENSHELL_POD_NAMESPACE + valueFrom: {fieldRef: {fieldPath: metadata.namespace}} + - {name: OPENSHELL_SERVICE_ACCOUNT_NAME, value: {{ printf "%s-gateway" .Release.Name | quote }}} + - {name: OPENSHELL_PEER_SERVICE_NAME, value: {{ printf "%s-gateway-peer" .Release.Name | quote }}} + - {name: OPENSHELL_PEER_TOKEN_AUDIENCE, value: openshell-gateway-peer} + - {name: OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE, value: /var/run/secrets/openshell-peer/token} + - {name: OPENSHELL_PEER_POD_LABELS, value: {{ printf "app.kubernetes.io/instance=%s,openshell.pattern/role=gateway" .Release.Name | quote }}} + - {name: OPENSHELL_PEER_TLS_SERVER_NAME, value: {{ printf "%s-gateway.%s.svc.cluster.local" .Release.Name .Release.Namespace | quote }}} + - {name: OPENSHELL_PEER_TLS_CA_FILE, value: /etc/openshell-tls/server/ca.crt} + - {name: OPENSHELL_TELEMETRY_ENABLED, value: "false"} + {{- if eq .Values.deployment.mode "kata" }} + # Use environment overrides so older pinned gateway builds can parse the stock TOML. + - {name: OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS, value: {{ $k.startupTimeouts.provisioningRepairWindowSeconds | quote }}} + - {name: OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS, value: {{ $k.startupTimeouts.runtimeBootstrapGraceSeconds | quote }}} + - {name: OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS, value: {{ $k.startupTimeouts.sandboxAttachRequestTimeoutSeconds | quote }}} + {{- end }} + - name: OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY + valueFrom: + secretKeyRef: {name: {{ $k.identity.credentialKeySecret | quote }}, key: key-encryption-key} + {{- if .Values.oidc.caConfigMap }} + - {name: SSL_CERT_FILE, value: /etc/openshell-tls/oidc-ca/ca.crt} + {{- end }} + ports: + - {name: gateway, containerPort: 17670} + - {name: health, containerPort: 8081} + startupProbe: + httpGet: {path: /healthz, port: health} + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: {path: /readyz, port: health} + periodSeconds: 5 + livenessProbe: + httpGet: {path: /healthz, port: health} + periodSeconds: 15 + resources: + {{- toYaml $k.gateway.resources | nindent 12 }} + volumeMounts: + - {name: data, mountPath: /var/openshell} + # Stock gateway refuses symlinked config: subPath plus checksum rollout. + - {name: config, mountPath: /etc/openshell/gateway.toml, subPath: gateway.toml, readOnly: true} + - {name: server-tls, mountPath: /etc/openshell-tls/server, readOnly: true} + - {name: client-ca, mountPath: /etc/openshell-tls/client-ca, readOnly: true} + - {name: jwt, mountPath: /etc/openshell-jwt, readOnly: true} + - {name: peer-token, mountPath: /var/run/secrets/openshell-peer, readOnly: true} + - {name: proxy-ca, mountPath: /etc/openshell-tls/proxy-ca, readOnly: true} + - {name: tmp, mountPath: /tmp} + {{- if .Values.oidc.caConfigMap }} + - {name: oidc-ca, mountPath: /etc/openshell-tls/oidc-ca, readOnly: true} + {{- end }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ $k.storage.existingGatewayClaim | default (printf "%s-gateway-data" .Release.Name) | quote }} + - {name: config, configMap: {name: {{ .Release.Name }}-gateway-config}} + - {name: server-tls, secret: {secretName: {{ $k.identity.serverTlsSecret | quote }}}} + - name: client-ca + secret: + secretName: {{ $k.identity.clientCaSecret | quote }} + items: [{key: ca.crt, path: ca.crt}] + - {name: jwt, secret: {secretName: {{ $k.identity.jwtSecret | quote }}, defaultMode: 0440}} + - {name: proxy-ca, configMap: {name: {{ $k.proxy.caConfigMap | quote }}}} + - {name: tmp, emptyDir: {}} + {{- if .Values.oidc.caConfigMap }} + - {name: oidc-ca, configMap: {name: {{ .Values.oidc.caConfigMap | quote }}}} + {{- end }} + - name: peer-token + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: {path: token, audience: openshell-gateway-peer, expirationSeconds: 3600} +{{- if not $k.storage.existingGatewayClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-gateway-data + annotations: + helm.sh/resource-policy: keep + argocd.argoproj.io/sync-options: Prune=false,Delete=false +spec: + accessModes: [ReadWriteOnce] + storageClassName: {{ $k.storage.className | quote }} + resources: + requests: {storage: {{ $k.storage.gatewaySize | quote }}} +{{- end }} diff --git a/charts/openshell-saw-kubernetes/templates/networkpolicy.yaml b/charts/openshell-saw-kubernetes/templates/networkpolicy.yaml new file mode 100644 index 000000000..902588a6b --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/networkpolicy.yaml @@ -0,0 +1,66 @@ +# Namespace owns one gateway/workspace family; trusted provider keys live elsewhere. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-default-deny +spec: + podSelector: {} + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-gateway +spec: + podSelector: + matchLabels: {app.kubernetes.io/instance: {{ .Release.Name }}, openshell.pattern/role: gateway} + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: {policy-group.network.openshift.io/ingress: ""} + ports: [{protocol: TCP, port: 17670}] + - from: + - podSelector: + matchLabels: + openshell.ai/boundary-role: supervisor + openshell.ai/gateway-id: {{ .Release.Name | quote }} + ports: [{protocol: TCP, port: 17670}] + - from: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: {{ .Values.kubernetes.network.reconcilerNamespace | quote }}} + podSelector: + matchLabels: + {{- toYaml .Values.kubernetes.network.reconcilerLabels | nindent 14 }} + ports: [{protocol: TCP, port: 17670}] + egress: + {{- toYaml .Values.kubernetes.network.gatewayEgress | nindent 4 }} +# Do not add workload egress here. Stock OpenShell owns its workload fence. +# Supervisor's stock allow-all egress is restricted by the separate native ANP. + +--- +# The pinned shared-namespace driver GETs and validates these two fences. +# Install them before the gateway starts so it never needs network policy writes. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: openshell-sandbox-workloads + labels: {openshell.ai/component: sandbox-workload-fence, openshell.ai/managed-by: openshell} +spec: + podSelector: {matchLabels: {openshell.ai/boundary-role: workload}} + policyTypes: [Ingress, Egress] + ingress: + - from: [{podSelector: {matchLabels: {openshell.ai/boundary-role: supervisor}}}] + ports: [{port: 5500, protocol: TCP}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: openshell-sandbox-supervisors + labels: {openshell.ai/component: sandbox-supervisor-egress, openshell.ai/managed-by: openshell} +spec: + podSelector: {matchLabels: {openshell.ai/boundary-role: supervisor}} + policyTypes: [Egress] + egress: [{}] diff --git a/charts/openshell-saw-kubernetes/templates/rbac.yaml b/charts/openshell-saw-kubernetes/templates/rbac.yaml new file mode 100644 index 000000000..fc5b56947 --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/rbac.yaml @@ -0,0 +1,81 @@ +{{/* Verbs follow stock shared-namespace driver at OpenShell 305c8b3. */}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-gateway +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-sandbox +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-gateway +rules: + - apiGroups: [agents.x-k8s.io] + resources: [sandboxes, sandboxes/status] + verbs: [create, delete, get, list, patch, update, watch] + - apiGroups: [""] + resources: [pods] + verbs: [create, delete, get, list, patch, watch] + - apiGroups: [""] + resources: [events] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [persistentvolumeclaims] + verbs: [get] + - apiGroups: [""] + resources: [services] + verbs: [create, get] + - apiGroups: [""] + resources: [secrets] + verbs: [create, delete] + - apiGroups: [""] + resources: [secrets] + resourceNames: [{{ .Values.kubernetes.identity.serverTlsSecret | quote }}] + verbs: [get] + - apiGroups: [networking.k8s.io] + resources: [networkpolicies] + # The stock driver creates and verifies per-sandbox os-* policies. + verbs: [get, create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-gateway +subjects: + - {kind: ServiceAccount, name: {{ .Release.Name }}-gateway, namespace: {{ .Release.Namespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ .Release.Name }}-gateway} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-gateway +rules: + - apiGroups: [authentication.k8s.io] + resources: [tokenreviews] + verbs: [create] + - apiGroups: [""] + resources: [nodes] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [namespaces] + resourceNames: [{{ .Release.Namespace | quote }}] + verbs: [get] + {{- if .Values.kubernetes.workload.runtimeClassName }} + - apiGroups: [node.k8s.io] + resources: [runtimeclasses] + resourceNames: [{{ .Values.kubernetes.workload.runtimeClassName | quote }}] + verbs: [get] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ .Release.Name }}-gateway +subjects: + - {kind: ServiceAccount, name: {{ .Release.Name }}-gateway, namespace: {{ .Release.Namespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ .Release.Namespace }}-{{ .Release.Name }}-gateway} diff --git a/charts/openshell-saw-kubernetes/templates/service.yaml b/charts/openshell-saw-kubernetes/templates/service.yaml new file mode 100644 index 000000000..44d5ac6ae --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/service.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-gateway +spec: + selector: {app.kubernetes.io/instance: {{ .Release.Name }}, openshell.pattern/role: gateway} + ports: [{name: gateway, port: 17670, targetPort: gateway}] +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-gateway-peer +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: {app.kubernetes.io/instance: {{ .Release.Name }}, openshell.pattern/role: gateway} + ports: [{name: gateway, port: 17670, targetPort: gateway}] +{{- if .Values.route.enabled }} +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ .Release.Name }}-gateway +spec: + {{- if .Values.route.host }} + host: {{ .Values.route.host | quote }} + {{- end }} + to: {kind: Service, name: {{ .Release.Name }}-gateway} + port: {targetPort: gateway} + tls: {termination: passthrough} +{{- end }} diff --git a/charts/openshell-saw-kubernetes/templates/validate.yaml b/charts/openshell-saw-kubernetes/templates/validate.yaml new file mode 100644 index 000000000..1f2f80134 --- /dev/null +++ b/charts/openshell-saw-kubernetes/templates/validate.yaml @@ -0,0 +1 @@ +{{- include "saw-kubernetes.validate" . -}} diff --git a/charts/openshell-saw-kubernetes/values.schema.json b/charts/openshell-saw-kubernetes/values.schema.json new file mode 100644 index 000000000..e7c5e2f90 --- /dev/null +++ b/charts/openshell-saw-kubernetes/values.schema.json @@ -0,0 +1,288 @@ +{ + "type": "object", + "additionalProperties": true, + "properties": { + "deployment": { + "type": "object", + "additionalProperties": false, + "properties": { + "mode": { + "type": "string", + "enum": [ + "container", + "kata" + ] + } + }, + "required": [ + "mode" + ] + }, + "kubernetes": { + "type": "object", + "additionalProperties": false, + "properties": { + "experimental": { + "type": "boolean" + }, + "startupTimeouts": { + "type": "object", + "additionalProperties": false, + "properties": { + "provisioningRepairWindowSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 86400 + }, + "runtimeBootstrapGraceSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 86400 + }, + "sandboxAttachRequestTimeoutSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 86400 + } + }, + "required": [ + "provisioningRepairWindowSeconds", + "runtimeBootstrapGraceSeconds", + "sandboxAttachRequestTimeoutSeconds" + ] + }, + "gateway": { + "type": "object", + "additionalProperties": false, + "properties": { + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.-]+)?$" + }, + "image": { + "type": "string" + }, + "resources": { + "type": "object" + } + }, + "required": [ + "version", + "image", + "resources" + ] + }, + "supervisor": { + "type": "object", + "additionalProperties": false, + "properties": { + "image": { + "type": "string" + } + }, + "required": [ + "image" + ] + }, + "workload": { + "type": "object", + "additionalProperties": false, + "properties": { + "image": { + "type": "string" + }, + "runtimeImage": { + "type": "string" + }, + "runtimeClassName": { + "type": "string", + "pattern": "^$|^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" + } + }, + "required": [ + "image", + "runtimeImage", + "runtimeClassName" + ] + }, + "identity": { + "type": "object", + "additionalProperties": false, + "properties": { + "serverTlsSecret": { + "type": "string" + }, + "clientCaSecret": { + "type": "string" + }, + "jwtSecret": { + "type": "string" + }, + "credentialKeySecret": { + "type": "string" + } + }, + "required": [ + "serverTlsSecret", + "clientCaSecret", + "jwtSecret", + "credentialKeySecret" + ] + }, + "storage": { + "type": "object", + "additionalProperties": false, + "properties": { + "className": { + "type": "string" + }, + "gatewaySize": { + "type": "string", + "pattern": "^[1-9][0-9]*(Mi|Gi|Ti)$" + }, + "workspaceSize": { + "type": "string", + "pattern": "^[1-9][0-9]*(Mi|Gi|Ti)$" + }, + "existingGatewayClaim": { + "type": "string" + } + }, + "required": [ + "className", + "gatewaySize", + "workspaceSize", + "existingGatewayClaim" + ] + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "proxy": { + "type": "object", + "additionalProperties": false, + "properties": { + "url": { + "type": "string" + }, + "caConfigMap": { + "type": "string" + } + }, + "required": [ + "url", + "caConfigMap" + ] + }, + "governance": { + "type": "object", + "additionalProperties": false, + "properties": { + "endpoint": { + "type": "string" + } + }, + "required": [ + "endpoint" + ] + }, + "network": { + "type": "object", + "additionalProperties": false, + "properties": { + "gatewayEgress": { + "type": "array" + }, + "reconcilerNamespace": { + "type": "string" + }, + "reconcilerLabels": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "gatewayEgress", + "reconcilerNamespace", + "reconcilerLabels" + ] + } + }, + "required": [ + "experimental", + "startupTimeouts", + "gateway", + "supervisor", + "workload", + "identity", + "storage", + "imagePullSecrets", + "proxy", + "governance", + "network" + ] + }, + "oidc": { + "type": "object", + "additionalProperties": false, + "properties": { + "issuerUrl": { + "type": "string" + }, + "clientId": { + "type": "string" + }, + "rolesClaim": { + "type": "string" + }, + "adminRole": { + "type": "string" + }, + "userRole": { + "type": "string" + }, + "caConfigMap": { + "type": "string" + } + }, + "required": [ + "issuerUrl", + "clientId", + "rolesClaim", + "adminRole", + "userRole", + "caConfigMap" + ] + }, + "route": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "host": { + "type": "string" + } + }, + "required": [ + "enabled", + "host" + ] + } + }, + "required": [ + "deployment", + "kubernetes", + "oidc", + "route" + ], + "$schema": "http://json-schema.org/draft-07/schema#" +} diff --git a/charts/openshell-saw-kubernetes/values.yaml b/charts/openshell-saw-kubernetes/values.yaml new file mode 100644 index 000000000..50781a176 --- /dev/null +++ b/charts/openshell-saw-kubernetes/values.yaml @@ -0,0 +1,66 @@ +deployment: + mode: container + +# Explicit opt-in: chart readiness is gateway readiness, not SAW feature parity. +kubernetes: + experimental: false + # Kata VMs can need more time than the stock OpenShell startup windows. + # The chart applies these values only when deployment.mode is kata. + startupTimeouts: + provisioningRepairWindowSeconds: 900 + runtimeBootstrapGraceSeconds: 900 + sandboxAttachRequestTimeoutSeconds: 900 + gateway: + version: 0.1.6-rhaiv.4 + image: quay.io/opendatahub/odh-openshell-gateway@sha256:bd2d7c9084b80b8d3d8ef92d00a27b963b35168fad84cd862b7f91fd9c488c3e + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} + supervisor: + image: quay.io/opendatahub/odh-openshell-supervisor@sha256:626b91a0f53393dac0d591c142b81100916aa66b4e65444b5527672b5533fe86 + workload: + # Application image; set to an approved immutable sandbox image. + image: "" + runtimeImage: quay.io/opendatahub/odh-openshell-sandbox@sha256:0cde11346e68485907c22d3c9b30802e52cf16e3853720a864ac8347bb68f6ef + runtimeClassName: "" + identity: + # Existing, retained identity; generated once outside Helm by its owner. + # server TLS: tls.crt,tls.key,ca.crt; client CA: ca.crt only mounted. + serverTlsSecret: "" + clientCaSecret: "" + # signing.pem, public.pem, kid + jwtSecret: "" + # key: key-encryption-key (stock gateway format) + credentialKeySecret: "" + storage: + className: "" + gatewaySize: 5Gi + workspaceSize: 20Gi + # Prefer an independently owned retained claim for existing state. + existingGatewayClaim: "" + imagePullSecrets: [] + # Only CA certificates are published into the runtime namespace. + proxy: + url: "" + caConfigMap: "" + # Reviewed stock governance endpoint, reached through declared egress. + governance: + endpoint: "" + network: + # All gateway egress is explicit; no namespace-wide internet allowance. + # Include API Service + real backend IPs, DNS, OIDC, governance and proxy. + gatewayEgress: [] + # Trusted API reconciler Pods are selected by exact namespace AND label. + reconcilerNamespace: "" + reconcilerLabels: {} + +oidc: + issuerUrl: "" + clientId: openshell-cli + rolesClaim: realm_access.roles + adminRole: openshell-admin + userRole: openshell-user + caConfigMap: "" +route: + enabled: true + host: "" diff --git a/charts/openshell-saw/files/installer/apply_bom.py b/charts/openshell-saw/files/installer/apply_bom.py index ec8673cb1..35d8b0ad7 100644 --- a/charts/openshell-saw/files/installer/apply_bom.py +++ b/charts/openshell-saw/files/installer/apply_bom.py @@ -65,7 +65,46 @@ import yaml -API_VERSION = "saw.redhat.com/v1alpha1" +# Helm embeds the canonical shared source in this signed file. The sentinel +# branch supports checked-out source/test fixtures only, by exact sibling path. +_SAW_CORE_EMBEDDED_B64 = "__SAW_CORE_SOURCE_BASE64__" +_SAW_ADMIN_EMBEDDED_B64 = "__SAW_ADMIN_SOURCE_BASE64__" +_SAW_UI_LIMIT_EMBEDDED_B64 = "__SAW_UI_LIMIT_SOURCE_BASE64__" + + +def _load_saw_module(name, embedded): + import types + sentinel = "__SAW_" + name.upper() + "_SOURCE_BASE64__" + if embedded == sentinel: + source_path = Path(__file__).resolve().with_name("saw_" + name + ".py") + if source_path.is_symlink(): + raise RuntimeError("shared SAW module must be the trusted adjacent source file") + source = source_path.read_bytes() + origin = str(source_path) + else: + source = base64.b64decode(embedded, validate=True) + origin = "" + module_name = "_saw_shared_" + name + "_" + hashlib.sha256(source).hexdigest() + cached = sys.modules.get(module_name) + if cached is not None: + return cached, source + module = types.ModuleType(module_name) + module.__file__ = origin + sys.modules[module_name] = module + try: + exec(compile(source, origin, "exec"), module.__dict__) + except BaseException: + sys.modules.pop(module_name, None) + raise + return module, source + + +_saw_core, _SAW_CORE_SOURCE_BYTES = _load_saw_module("core", _SAW_CORE_EMBEDDED_B64) +_saw_admin, _SAW_ADMIN_SOURCE_BYTES = _load_saw_module("admin", _SAW_ADMIN_EMBEDDED_B64) +_saw_ui_limit, _SAW_UI_LIMIT_SOURCE_BYTES = _load_saw_module("ui_limit", _SAW_UI_LIMIT_EMBEDDED_B64) +globals().update({name: getattr(_saw_core, name) for name in _saw_core.__all__}) + + INSTALLER_VERSION = "0.1.0" DEFAULT_INPUTS = Path("/run/saw") @@ -96,10 +135,6 @@ DIGEST_IMAGE_RE = re.compile(r"^[a-z0-9][a-z0-9./:_-]*@sha256:[0-9a-f]{64}$") IMAGE_RE = re.compile(r"^[a-z0-9][a-z0-9./:_@-]*$") VERSION_RE = re.compile(r"^v?[0-9]+\.[0-9]+\.[0-9]+[A-Za-z0-9.+_-]*$") -NAME_RE = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") -SECRET_NAME_RE = re.compile(r"^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$") -SECRET_KEY_RE = re.compile(r"^[-._a-zA-Z0-9]+$") -OPENSHELL_NAME_LIMIT = 19 # The installer's own mTLS identity; the gateway reads roles from the OU. ADMIN_CERT_SUBJECT = "/O=openshell/OU=openshell-admin/CN=saw-installer" ALREADY_RE = re.compile(r"already (exists|a member)", re.IGNORECASE) @@ -116,115 +151,6 @@ REFRESH_BOOTSTRAP_CREDENTIAL = "saw-refresh-pending" RESTART_MODE_HINT = "run make saw-restart-pending CONFIRM=yes to boot the new credential mode" -PROVIDER_CRED_MAP = { - "gemini": "GEMINI_API_KEY", - "google-vertex-ai": "GOOGLE_API_KEY", - "claude-code": "ANTHROPIC_API_KEY", - "anthropic": "ANTHROPIC_API_KEY", - "codex": "OPENAI_API_KEY", - "openai": "OPENAI_API_KEY", - "nvidia": "NVIDIA_API_KEY", - "build": "NVIDIA_INFERENCE_API_KEY", - "brave": "BRAVE_API_KEY", - "tavily": "TAVILY_API_KEY", -} - -# NemoClaw names the NVIDIA API Catalog `build`, OpenShell's profile for it -# is `nvidia`. The BOM uses OpenShell's name; the installer translates for -# `nemoclaw onboard`. `nemoclawProvider` on a provider overrides this. -NEMOCLAW_PROVIDER_MAP = { - "nvidia": "build", -} - - -def nemoclaw_provider(provider): - return provider.nemoclaw_provider or NEMOCLAW_PROVIDER_MAP.get(provider.type, provider.type) - - -def profile_credential_envs(profile_text): - """The first env var of each credential a provider profile declares, in - order (credentials[i].env_vars[0]); "" for a credential without one.""" - try: - doc = yaml.safe_load(profile_text) if profile_text else None - except yaml.YAMLError: - return [] - out = [] - for cred in (doc or {}).get("credentials") or []: - envs = (cred.get("env_vars") or []) if isinstance(cred, dict) else [] - out.append(envs[0] if envs else "") - return out - - -def profile_credentials(profile_text): - """A provider profile's credentials: [(name, [env vars])], in order.""" - try: - doc = yaml.safe_load(profile_text) if profile_text else None - except yaml.YAMLError: - return [] - out = [] - for cred in (doc or {}).get("credentials") or []: - if isinstance(cred, dict) and cred.get("name"): - out.append((str(cred["name"]), [str(e) for e in cred.get("env_vars") or []])) - return out - - -def profile_all_credential_envs(profile_text): - """Every env var any credential of a provider profile is exposed under - (all of credentials[].env_vars, e.g. GITHUB_TOKEN and GH_TOKEN).""" - try: - doc = yaml.safe_load(profile_text) if profile_text else None - except yaml.YAMLError: - return set() - return {env for cred in (doc or {}).get("credentials") or [] if isinstance(cred, dict) - for env in (cred.get("env_vars") or [])} - - -def profile_credential_env(profile_text): - """The primary credential env var a provider profile declares - (credentials[0].env_vars[0]), or "" when it declares none.""" - envs = profile_credential_envs(profile_text) - return envs[0] if envs else "" - - -def credential_env_name(provider_type, profiles): - """Env var the installer passes a provider's credential through: the one - the provider's own profile declares, else the built-in default. This lets - a profile-backed type (github, slack, gmail, ...) work as a provider with - no PROVIDER_CRED_MAP edit; for the shipped built-ins the two agree.""" - return (profile_credential_env((profiles or {}).get(provider_type, "")) - or PROVIDER_CRED_MAP.get(provider_type, "")) - - -SANDBOX_TYPES = {"generic", "openclaw", "nemoclaw"} -# Provider config key that records a provider's base URL. OpenShell 0.1.x has -# no inference router: the agent calls the endpoint itself (see -# NATIVE_BASE_URLS), and the provider profile must name that endpoint's host. -BASE_URL_CONFIG_KEYS = { - "openai": "OPENAI_BASE_URL", - "anthropic": "ANTHROPIC_BASE_URL", - "nvidia": "NVIDIA_BASE_URL", -} -# OpenShell 0.1.x removed managed inference (`openshell inference`, -# https://inference.local). An agent is configured with its provider's native -# OpenAI-compatible endpoint and the placeholder key the sandbox receives in -# the provider's env var; the sandbox proxy swaps in the real key only for -# requests to the profile's endpoints from the profile's binaries. A -# provider's baseUrl overrides these. -NATIVE_BASE_URLS = { - "nvidia": "https://integrate.api.nvidia.com/v1", - "build": "https://integrate.api.nvidia.com/v1", - "openai": "https://api.openai.com/v1", -} - -OPENCLAW_AUTH_CHOICES = {"openai": "openai-api-key", "anthropic": "apiKey"} -# By protocol: the official API an agent talks to when no endpoint is given. -PROTOCOL_NATIVE_BASE_URLS = {"openai": "https://api.openai.com/v1", - "anthropic": "https://api.anthropic.com"} - - -class InstallerError(Exception): - """A failure with a message that is safe to log and show in status.""" - # --------------------------------------------------------------------------- # Logging @@ -664,111 +590,11 @@ def check_trusted_proxy(tp): # --------------------------------------------------------------------------- -# Profiles +# VM profile inputs and harness preparation (shared parsing is in saw_core) # --------------------------------------------------------------------------- -@dataclass -class Provider: - name: str - type: str - enabled: bool = True - nemoclaw_provider: str = "" - credential_secret: str = "" - credential_secret_key: str = "api_key" - model: str = "" - # Secret keys the base URL and the model are read from (optional). - base_url_secret_key: str = "" - model_secret_key: str = "" - inference_timeout: int = 0 - base_url: str = "" - credential_free: bool = False - # Optional `credentials:` mapping, keyed by the profile's credential name - # ({api_key: {secretKey: ...}, app_secret: {secretKey: ...}}): one value - # per credential, from the same Secret, each passed as its own - # `--credential ENV` with the env var that credential declares. The - # single-credential shorthand credentialSecretKey is one entry for the - # profile's first credential. - refresh_strategy: str = "" - refresh_material: dict = field(default_factory=dict) - credentials_spec: dict = field(default_factory=dict) - credential_values: dict = field(default_factory=dict) # credential name -> value - -@dataclass -class Sandbox: - name: str - type: str = "generic" - enabled: bool = True - agent: str = "openclaw" - image: str = "" - providers: list = field(default_factory=list) - model: str = "" - harness_ref: dict = field(default_factory=dict) - # `env:` -- the workload's environment (`sandbox create --env`; changing - # it recreates the sandbox). {NAME: {value, title?}}: literals only, so - # env_values (below) is derived, never a resolved copy that could go - # stale (found in CI: freshly parsed profiles compared an empty copy - # with the sandbox's env label and recreated it). Never for credentials - # (the agent, and argv, can read these): those stay on providers. - env: dict = field(default_factory=dict) - # `inference:` -- how the agent reaches its model: {providerRef, protocol, - # baseUrl?, model, models?, timeoutSeconds?}. models maps optional native - # provider names to additional chat model inputs. baseUrl/model/timeoutSeconds - # is a literal or an input ({secretKeyRef: {name, key}, default?, - # required?, title?}); inference_values is it resolved. The installer onboards the - # agent from it (by protocol, so any profile-backed provider type works). - inference: dict = field(default_factory=dict) - inference_values: dict = field(default_factory=dict) - - # Container-level GPU request (`openshell sandbox create --gpu`). - # Independent of the gateway VM's vm.gpu.enabled (KubeVirt hostDevices). - # Off unless the BOM sandbox sets gpu.enabled. See docs/gpu-passthrough.md. - gpu_enabled: bool = False - gpu_count: int = 1 - - @property - def env_values(self): - """NAME -> value for the sandbox's env (literal entries; anything - else is refused by validation).""" - if not isinstance(self.env, dict): - return {} - return {str(name): str(src["value"]) for name, src in sorted(self.env.items()) - if isinstance(src, dict) and "value" in src} - - -@dataclass -class Workspace: - name: str - enabled: bool = True - description: str = "" - providers: list = field(default_factory=list) - sandboxes: list = field(default_factory=list) - - -@dataclass -class Profile: - name: str - workspaces: list = field(default_factory=list) - - -@dataclass -class HarnessBundle: - name: str - agent: str = "openclaw" - digest: str = "" - files: dict = field(default_factory=dict) # relpath -> base64 text - - -def tree_digest(files): - """Shared contract: sha256 over sorted "\\0\\n" lines. - - `files` maps a POSIX relative path to file bytes. Helm computes the same - value at render time; templates/configmap-bom.yaml must agree. - """ - digest = hashlib.sha256() - for rel in sorted(files): - digest.update(f"{rel}\x00{hashlib.sha256(files[rel]).hexdigest()}\n".encode()) - return "sha256:" + digest.hexdigest() +tree_digest = _saw_core.tree_digest # -- Harness bundles --------------------------------------------------------- @@ -824,7 +650,6 @@ def tree_digest(files): # mount; how podman reports the mount (see sandbox_harness_mount). The 0.1.x # admission rules above are from the v0.1.2 source. -HARNESS_MOUNT = "/sandbox/harness" HARNESS_MARKER = ".saw-harness-revision" # written into the volume with the bundle HARNESS_VOLUME_PREFIX = "saw-harness-" HARNESS_VOLUME_LABEL = "saw.redhat.com/harness-volume" @@ -844,16 +669,6 @@ def tree_digest(files): OPENCLAW_CHAT_CHANNELS = frozenset({"slack"}) OPENCLAW_EXEC_ENV = ("OPENCLAW_HOME=/sandbox SQLITE_TMPDIR=/sandbox/.openclaw/state " "TMPDIR=/sandbox/.openclaw/state OPENCLAW_NIX_MODE=0") -# mcp.json env/header value: exactly ${VAR}, or Authorization-style -# "Bearer ${VAR}". Any other prefix could smuggle a literal secret. -MCP_PLACEHOLDER_ONLY = re.compile( - r"^(?:\$\{[A-Za-z_][A-Za-z0-9_]*\}|Bearer \$\{[A-Za-z_][A-Za-z0-9_]*\})$", - re.IGNORECASE) -# A stdio arg naming a credential-ish flag with a literal value, e.g. -# "--api-key=sk-live-...". The "--flag value" two-token form is not covered. -MCP_ARG_CREDENTIAL_RE = re.compile( - r"^--?[\w-]*(?:key|token|secret|password|passwd|auth)[\w-]*=(?P.*)$", - re.IGNORECASE) # The podman driver's workload container (its supervisor has the same # sandbox labels but not the user's mounts): isolation.rs WORKLOAD_FILTER. WORKLOAD_ROLE_LABEL = "openshell.ai/isolation-role=sandbox" @@ -901,12 +716,6 @@ def sandbox_mounts_json(ws, sb): return json.dumps({"podman": {"mounts": sandbox_mounts(ws, sb)}}, separators=(",", ":")) -def safe_rel_path(rel, where): - """A bundle-relative POSIX path: no absolute, no empty or '.'/'..' segment.""" - parts = rel.split("/") - if not rel or rel.startswith("/") or any(part in ("", ".", "..") for part in parts): - raise InstallerError(f"{where}: unsafe path {rel!r}") - return rel def read_harness_tar(path): @@ -1007,143 +816,7 @@ def read_volume_tree(root): def describe_harness_tree(files, inline=False): - """What OpenClaw has to be pointed at, and what governance must allow. - - Reads harness.yaml (metadata, spec.agent, and the governance entries in - spec.plugins / spec.mcpServers) and mcp.json. OpenClaw never reads - harness.yaml; it is the installer's view of the bundle. Returns: - name, agent - skills the bundle has skills/ - agentPluginsBundle plugin.json at the root (OpenClaw loads skills/ and - mcp.json from the bundle root) - mcp the bundle has mcp.json - mcpServers server names declared in mcp.json - pluginDirs native plugins under plugins/ - governance [{kind, name, governanceProfile, hosts}] to check - against the gateway's catalog and the sandbox's - providers - - Keys never travel in a bundle. A plugin or MCP server that calls a - service names the provider profile (governanceProfile) that governs it; - the sandbox must have a provider of that type, which gives the process - the profile's env var holding a placeholder, and the sandbox's egress - proxy puts the real key in only for the profile's endpoints and - binaries. `inline` bundles come from a ConfigMap, which keeps no file - modes, so a stdio server cannot run a bundled file directly. - """ - doc = _yaml(files["harness.yaml"][0].decode("utf-8"), "harness.yaml") - spec = doc.get("spec") or {} - name = (doc.get("metadata") or {}).get("name", "") - mcp_server_names = [] - mcp_configs = {} - if "mcp.json" in files and "plugin.json" not in files: - raise InstallerError( - f"harness '{name}': mcp.json with no plugin.json at the bundle root; OpenClaw only " - "reads a bundle's mcp.json when the bundle root is on plugins.load.paths, which " - "openclaw_harness_config only sets for an Agent Plugins bundle. Add a plugin.json " - "at the bundle root") - plugin_dirs = sorted({rel.split("/")[1] for rel in files - if rel.startswith("plugins/") and rel.count("/") >= 2}) - declared_plugins = {item.get("name") for item in spec.get("plugins") or []} - for plugin in plugin_dirs: - if plugin not in declared_plugins: - log(f"WARN: harness '{name}': plugin '{plugin}' has no entry in harness.yaml " - "spec.plugins; its egress, if any, is enforced at runtime by the sandbox " - "proxy, not checked here") - governance = [] - for item in spec.get("plugins") or []: - if item.get("governanceProfile"): - governance.append({"kind": "plugin", "name": item.get("name", ""), - "governanceProfile": item["governanceProfile"], "hosts": []}) - declared = {m.get("name"): m for m in spec.get("mcpServers") or []} - for server, decl in sorted(declared.items()): - stale = sorted(k for k in ("credentialSecret", "credentialSecretKey", "credentialEnvVar") - if k in decl) - if stale: - raise InstallerError( - f"harness '{name}': mcp server '{server}' sets {', '.join(stale)}; keys no longer " - "reach the sandbox that way. Set governanceProfile to the provider type whose " - "key the server needs and give the sandbox a provider of that type: the server " - "reads a placeholder from the provider's env var (for example BRAVE_API_KEY) " - "and the sandbox's egress proxy adds the real key") - if "mcp.json" in files: - try: - servers = json.loads(files["mcp.json"][0]).get("mcpServers") or {} - except ValueError as exc: - raise InstallerError(f"harness '{name}': mcp.json is not valid JSON ({exc})") from None - mcp_server_names = sorted(servers) - mcp_configs = {k: v for k, v in servers.items() if isinstance(v, dict)} - for server, conf in sorted(servers.items()): - if not isinstance(conf, dict) or conf.get("type") not in ("stdio", "streamable-http", "sse"): - raise InstallerError( - f"harness '{name}': mcp.json server '{server}' needs \"type\": \"stdio\", " - "\"streamable-http\" or \"sse\" (OpenClaw ignores it otherwise)") - for field in ("env", "headers"): - for key, value in (conf.get(field) or {}).items(): - if not isinstance(value, str) or not MCP_PLACEHOLDER_ONLY.match(value): - raise InstallerError( - f"harness '{name}': mcp.json server '{server}' {field} '{key}' is not " - "a bare ${VAR} placeholder (e.g. \"Bearer ${VAR}\"); a literal secret " - "must not ship in a bundle's mcp.json, the ConfigMap or the image") - for arg in conf.get("args") or []: - match = isinstance(arg, str) and MCP_ARG_CREDENTIAL_RE.match(arg) - if match and not MCP_PLACEHOLDER_ONLY.match(match.group("value")): - raise InstallerError( - f"harness '{name}': mcp.json server '{server}' arg {arg!r} looks like a " - "credential flag with a literal value; use --flag=${VAR} instead") - if isinstance(conf.get("url"), str): - creds = urlsplit(conf["url"]) - if creds.username or creds.password: - raise InstallerError( - f"harness '{name}': mcp.json server '{server}' url has a literal " - "credential in it (user:pass@host); a literal secret must not ship " - "in a bundle's mcp.json, the ConfigMap or the image") - profile = (declared.get(server) or {}).get("governanceProfile", "") - if conf["type"] == "stdio": - command = conf.get("command") - if inline and isinstance(command, str) and "${PLUGIN_ROOT}" in command: - raise InstallerError( - f"harness '{name}': mcp server '{server}' runs {command!r}, a file in the " - "bundle; an inline bundle keeps no file modes, so run it through its " - "interpreter (for example command: node, args: [${PLUGIN_ROOT}/...]) " - "or ship the bundle as an image") - # A stdio server runs inside the sandbox, under its policy. It - # needs a governanceProfile only when it calls a service. - if profile: - governance.append({"kind": "MCP server", "name": server, - "governanceProfile": profile, "hosts": []}) - else: - log(f"WARN: harness '{name}': stdio MCP server '{server}' has no " - "governanceProfile; its egress, if any, is enforced at runtime by " - "the sandbox proxy, not checked here") - continue - host = urlsplit(conf.get("url") or "").hostname - if not host: - raise InstallerError(f"harness '{name}': mcp.json server '{server}' has no url host") - if not profile: - raise InstallerError( - f"harness '{name}': remote MCP server '{server}' ({host}) needs a " - "governanceProfile in harness.yaml spec.mcpServers") - governance.append({"kind": "MCP server", "name": server, - "governanceProfile": profile, "hosts": [host]}) - bundle_id = name - if "plugin.json" in files: - try: - bundle_id = json.loads(files["plugin.json"][0].decode("utf-8")).get("name") or name - except (ValueError, AttributeError): - pass - return { - "name": name, - "bundleId": bundle_id, - "agent": spec.get("agent", "openclaw"), - "skills": any(rel.startswith("skills/") for rel in files), - "agentPluginsBundle": "plugin.json" in files, - "mcp": "mcp.json" in files, - "mcpServers": mcp_server_names, - "mcpConfigs": mcp_configs, - "pluginDirs": plugin_dirs, - "governance": governance, - } + return _saw_core.describe_harness_tree(files, inline, warn=log) def parse_provider_profile_catalog(output): @@ -1177,106 +850,17 @@ def in_cluster_host(host): return len(labels) >= 3 and (labels[2:3] == ["svc"] and labels[3:] in ([], ["cluster", "local"])) -def host_allowed(host, patterns): - """A profile endpoint host may be a glob (*-aiplatform.googleapis.com); - ports are not part of the match. Each `*` is one DNS label (no dots).""" - host = (host or "").split(":")[0].lower() - for pattern in patterns: - regex = re.escape((pattern or "").split(":")[0].lower()).replace(r"\*", r"[^.]*") - if re.fullmatch(regex, host): - return True - return False +host_allowed = _saw_core.host_allowed -def openclaw_harness_config(info): - """OpenClaw config that makes it load the mounted bundle. - An Agent Plugins bundle (plugin.json at the root) brings its skills/ and - mcp.json (MCP servers) with it, so the bundle root goes on - plugins.load.paths. Otherwise skills/ is an extra skill directory. - plugins/ holds native OpenClaw plugins (tool code); OpenClaw discovers - every plugin under it from the one parent path, so adding or removing a - plugin in the bundle needs no config change (checked live on OpenClaw - 2026.9.5). - """ - config = {} - paths = [] - if info["agentPluginsBundle"]: - paths.append(HARNESS_MOUNT) - elif info["skills"]: - config["skills.load.extraDirs"] = [f"{HARNESS_MOUNT}/skills"] - if info["pluginDirs"]: - paths.append(f"{HARNESS_MOUNT}/plugins") - if paths: - config["plugins.load.paths"] = paths - return config - - -# OpenClaw (2026.9.x) expands only ${PLUGIN_ROOT}, ${PLUGIN_DATA} and -# ${CLAUDE_PLUGIN_ROOT} in a bundle's mcp.json; any other ${VAR} reaches the -# server as literal text, and a stdio server inherits only HOME, LOGNAME, -# PATH, SHELL, TERM and USER. So a bundle server never sees its provider's -# placeholder (Slack answers invalid_auth) or the sandbox's CA settings (TLS -# through the egress proxy fails). openclaw.json does expand ${VAR} from the -# gateway's environment, which has both, and a server defined under -# mcp.servers replaces the bundle's server of the same name (found live: the -# daily-briefing Slack reader works only this way). -BUNDLE_ROOT_PLACEHOLDERS = ("${CLAUDE_PLUGIN_ROOT}", "${PLUGIN_ROOT}") -# The CA settings OpenShell gives a sandbox's processes when it has the proxy -# CA (openshell-sandbox child_env::tls_env_vars). Only the ones the sandbox -# really has are passed on: OpenClaw leaves an unset ${VAR} as literal text, -# and REQUESTS_CA_BUNDLE='${REQUESTS_CA_BUNDLE}' breaks every HTTPS call. -MCP_TRUST_ENV = ("NODE_EXTRA_CA_CERTS", "DENO_CERT", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", - "CURL_CA_BUNDLE", "GIT_SSL_CAINFO") -MCP_SERVER_NAME_RE = re.compile(r"^[A-Za-z0-9_-]+$") + +MCP_TRUST_ENV = _saw_core.MCP_TRUST_ENV SAW_MCP_STATE = "/sandbox/.openclaw/saw-mcp-servers.json" def harness_mcp_servers(info, trust_env=()): - """{name: mcp.servers entry} re-declaring the bundle's MCP servers in - openclaw.json, where OpenClaw expands ${VAR}. Stdio servers also get the - CA settings named in trust_env (the ones the sandbox has), so they trust - the egress proxy. Paths resolve as OpenClaw resolves them for a bundle - (root: the harness mount).""" - if info is None or not info.get("agentPluginsBundle"): - return {} - root = HARNESS_MOUNT - data = f"/sandbox/.openclaw/saw-plugin-data/{info['bundleId']}" - - def expand(value): - if not isinstance(value, str): - return value - for placeholder in BUNDLE_ROOT_PLACEHOLDERS: - value = value.replace(placeholder, root) - return value.replace("${PLUGIN_DATA}", data) - - out = {} - for name, conf in sorted((info.get("mcpConfigs") or {}).items()): - if not MCP_SERVER_NAME_RE.match(name): - log(f"WARN: harness '{info['name']}': MCP server name {name!r} cannot be " - "re-declared in openclaw.json; it keeps the bundle's literal env") - continue - kind = conf.get("type") - if kind == "stdio": - command = conf.get("command") - if isinstance(command, str) and command.startswith("./"): - command = f"{root}/{command[2:]}" - entry = {"command": expand(command), - "args": [expand(a) for a in conf.get("args") or []], - "env": {"PLUGIN_ROOT": root, "PLUGIN_DATA": data, - **{var: f"${{{var}}}" for var in trust_env}, - **{k: expand(v) for k, v in (conf.get("env") or {}).items()}}} - if conf.get("cwd"): - entry["cwd"] = expand(conf["cwd"]) - elif kind in ("streamable-http", "sse"): - if not conf.get("headers"): - continue # nothing for OpenClaw to expand: the bundle's own entry works - entry = {"url": expand(conf.get("url")), "transport": kind, - "headers": {k: expand(v) for k, v in conf["headers"].items()}} - else: - continue - out[name] = entry - return out + return _saw_core.harness_mcp_servers(info, trust_env, warn=log) def mcp_trust_env_probe(): @@ -1286,260 +870,15 @@ def mcp_trust_env_probe(): return f'for v in {names}; do [ -n "$(printenv "$v")" ] && echo "$v"; done; true' -def harness_plugin_ids(info): - """The OpenClaw plugin ids the harness brings: the bundle's (plugin.json - name) and each native plugin under plugins/.""" - ids = [info["bundleId"]] if info["agentPluginsBundle"] else [] - return ids + list(info["pluginDirs"]) -def mcp_registration_failures(output, declared): - """Inspect reports configuration registration, never MCP process readiness. +mcp_registration_failures = _saw_core.mcp_registration_failures - Kept pure so the E2E script can apply the same checks to inspection JSON. - Names in descriptions or diagnostics are not evidence of registration. - """ - try: - report = json.loads(output) - plugin = report["plugin"] - servers = report["mcpServers"] - diagnostics = report.get("diagnostics") or [] - if not isinstance(plugin, dict) or not isinstance(servers, list) or not isinstance(diagnostics, list): - raise ValueError("invalid inspection shape") - if plugin.get("status") != "loaded" or plugin.get("enabled") is False or plugin.get("error"): - return ["OpenClaw bundle runtime inspection reports an error or disabled bundle"] - if any(isinstance(d, dict) and d.get("level") == "error" for d in diagnostics): - return ["OpenClaw bundle runtime inspection reports error diagnostics"] - registered = {entry["name"] for entry in servers - if isinstance(entry, dict) and isinstance(entry.get("name"), str) - and not entry.get("unsupported")} - except (ValueError, TypeError, KeyError): - return ["could not parse OpenClaw MCP registration inspection JSON"] - missing = sorted(set(declared) - registered) - return ([f"OpenClaw does not list MCP server(s) {', '.join(missing)} as registered"] - if missing else []) - - -def parse_harness_files(files): - """Build bundles from flat ConfigMap keys harness____, plus - one harness____map key per bundle mapping each hash to its - relpath (content-addressed: the key no longer encodes the path, so it - never hits the iso9660/Joliet 64-character filename limit regardless of - how deep or long a bundle's paths are; see templates/configmap-bom.yaml). - - The digest is computed, never authored here: the pin lives in - sandbox.yaml harnessRef, outside the hashed tree. Relpaths from the map - get the same .. / empty / absolute checks as read_harness_tar. - """ - raw_by_bundle, maps = {}, {} - for key, raw in files.items(): - parts = key.split("__") - if len(parts) != 3 or parts[0] != "harness": - raise InstallerError(f"unexpected harness file name: {key}") - bundle, token = parts[1], parts[2] - if token == "map": - try: - maps[bundle] = json.loads(raw.decode("utf-8")) - except (ValueError, UnicodeDecodeError) as exc: - raise InstallerError( - f"harness bundle {bundle!r}: map is not valid JSON ({exc})") from None - else: - raw_by_bundle.setdefault(bundle, {})[token] = raw - bundles = {} - for name in sorted(raw_by_bundle): - path_map = maps.get(name) - if path_map is None: - raise InstallerError(f"harness bundle {name!r} has no path map (harness__{name}__map)") - tree = {} - for hash_, raw in raw_by_bundle[name].items(): - rel = path_map.get(hash_) - if rel is None: - raise InstallerError( - f"harness bundle {name!r}: key 'harness__{name}__{hash_}' is not in its path map") - tree[safe_rel_path(rel, f"harness bundle {name!r} path map")] = raw - if "harness.yaml" not in tree: - raise InstallerError(f"harness bundle {name!r} has no harness.yaml") - doc = _yaml(tree["harness.yaml"].decode("utf-8"), f"harness/{name}/harness.yaml") - spec = doc.get("spec") or {} - bundles[name] = HarnessBundle( - name=(doc.get("metadata") or {}).get("name", name), - agent=spec.get("agent", "openclaw"), - digest=tree_digest(tree), - files={rel: base64.b64encode(raw).decode() for rel, raw in tree.items()}, - ) - return bundles - - -def read_profile_files(directory): - """Read the flattened saw-bom ConfigMap. - - Returns (profile files as text, harness files as bytes, harness index). - Profile keys are profiles______.yaml; harness keys are - harness____ and hold base64 (see the digest - contract in templates/configmap-bom.yaml and parse_harness_files). - """ - directory = Path(directory) - if not directory.is_dir(): - return {}, {}, {} - files, harness, index = {}, {}, {} - for entry in sorted(directory.iterdir()): - if entry.name.startswith(".") or not entry.is_file(): - continue # kubelet/ISO housekeeping entries - if entry.name.startswith("profiles__") and entry.name.endswith(".yaml"): - files[entry.name] = entry.read_text(encoding="utf-8") - elif entry.name.startswith("harness__"): - try: - harness[entry.name] = base64.b64decode( - entry.read_text(encoding="utf-8"), validate=True) - except (ValueError, UnicodeDecodeError) as exc: - raise InstallerError( - f"harness file {entry.name} is not valid base64 ({exc}); " - "the chart must b64enc every harness key") from None - elif entry.name == "harness-index.yaml": - index = _yaml(entry.read_text(encoding="utf-8"), entry.name) - else: - # A wrong key layout must not look like "no profiles". - raise InstallerError(f"unexpected file in the profiles ConfigMap: {entry.name}") - return files, harness, index -def _yaml(text, where): - try: - return yaml.safe_load(text) or {} - except yaml.YAMLError as exc: - raise InstallerError(f"{where}: invalid YAML ({exc})") from None -def parse_profiles(files): - """Build profiles from {flat key: yaml text}. - Only keys of the form profiles______.yaml are - used, where is workspace, providers or sandbox. - """ - grouped = {} - for key, text in files.items(): - parts = key.split("__") - if len(parts) != 4 or parts[0] != "profiles": - raise InstallerError(f"unexpected profile file name: {key}") - _, profile, ws_dir, filename = parts - if filename not in ("workspace.yaml", "providers.yaml", "sandbox.yaml"): - raise InstallerError(f"unexpected profile file: {key}") - grouped.setdefault(profile, {}).setdefault(ws_dir, {})[filename] = (key, text) - - profiles = [] - for profile_name in sorted(grouped): - profile = Profile(name=profile_name) - for ws_dir in sorted(grouped[profile_name]): - docs = grouped[profile_name][ws_dir] - if "workspace.yaml" not in docs: - raise InstallerError(f"profile {profile_name}/{ws_dir} has no workspace.yaml") - key, text = docs["workspace.yaml"] - ws_doc = _yaml(text, key) - meta = ws_doc.get("metadata") or {} - spec = ws_doc.get("spec") or {} - ws = Workspace(name=meta.get("name", ws_dir), - enabled=spec.get("enabled", True), - description=meta.get("description", "")) - if "providers.yaml" in docs: - key, text = docs["providers.yaml"] - for p in (_yaml(text, key).get("spec") or {}).get("providers") or []: - if "name" not in p or "type" not in p: - raise InstallerError(f"{key}: every provider needs name and type") - cred_spec = p.get("credentials") - if cred_spec is not None and "credentialSecretKey" in p: - raise InstallerError(f"{key}: provider '{p['name']}' sets both " - "credentials and credentialSecretKey; use one") - primary_key = p.get("credentialSecretKey", "api_key") - ws.providers.append(Provider( - name=p["name"], type=p["type"], - enabled=p.get("enabled", True), - credential_free=p.get("credentialFree", False), - nemoclaw_provider=p.get("nemoclawProvider", ""), - credential_secret=p.get("credentialSecret", ""), - credential_secret_key=primary_key, - model=p.get("model", ""), - base_url_secret_key=p.get("baseUrlSecretKey", ""), - model_secret_key=p.get("modelSecretKey", ""), - inference_timeout=int(p.get("inferenceTimeout", 0) or 0), - refresh_strategy=str((p.get("refresh") or {}).get("strategy", "")), - credentials_spec=cred_spec if cred_spec is not None else {})) - if "sandbox.yaml" in docs: - key, text = docs["sandbox.yaml"] - for s in (_yaml(text, key).get("spec") or {}).get("sandboxes") or []: - if "name" not in s: - raise InstallerError(f"{key}: every sandbox needs a name") - gpu = s.get("gpu", {}) - if not isinstance(gpu, dict): - raise InstallerError(f"{key}: sandbox '{s['name']}' gpu must be a mapping") - unknown = sorted(set(gpu) - {"enabled", "count"}) - if unknown: - raise InstallerError(f"{key}: sandbox '{s['name']}' gpu has unknown keys {', '.join(unknown)}") - if not isinstance(gpu.get("enabled", False), bool): - raise InstallerError(f"{key}: sandbox '{s['name']}' gpu.enabled must be true or false") - gpu_count = gpu.get("count", 1) - if isinstance(gpu_count, bool) or not isinstance(gpu_count, int): - raise InstallerError(f"{key}: sandbox '{s['name']}' gpu.count must be an integer") - ws.sandboxes.append(Sandbox( - name=s["name"], type=s.get("type", "generic"), - enabled=s.get("enabled", True), - agent=s.get("agent", "openclaw"), - image=s.get("image", ""), - providers=list(s.get("providers") or []), - model=s.get("model", ""), - harness_ref=dict(s.get("harnessRef") or {}), - env=s.get("env") if s.get("env") is not None else {}, - inference=s.get("inference") if s.get("inference") is not None else {}, - gpu_enabled=gpu.get("enabled", False), - gpu_count=gpu_count)) - - profile.workspaces.append(ws) - profiles.append(profile) - return profiles - - -def enabled_workspaces(profiles): - for profile in profiles: - for ws in profile.workspaces: - if ws.enabled: - yield profile, ws - - -def pinned_bundle(sb, bundles): - """The inline bundle a sandbox's harnessRef.name points at. The bundle - and the pin ship in the same ConfigMap, so harnessRef.digest is - optional; when it is set it must match.""" - ref = sb.harness_ref or {} - bundle = bundles.get(ref.get("name")) - if bundle is None: - raise InstallerError(f"sandbox '{sb.name}' references unknown harness bundle " - f"'{ref.get('name')}'") - want = ref.get("digest") or "" - if want and want != bundle.digest: - raise InstallerError( - f"harnessRef digest mismatch for sandbox '{sb.name}': sandbox.yaml says {want}, " - f"the bundle hashes to {bundle.digest}") - return bundle - - -def check_harness_index(bundles, index): - """harness-index.yaml holds the digest Helm computed for each shipped - bundle; the installer recomputes it from the files it read. A mismatch - means the chart and the installer disagree on the tree_digest contract - or on the files (an encoding change on the way), so nothing is trusted.""" - listed = (index or {}).get("bundles") or {} - for name, digest in sorted(listed.items()): - bundle = bundles.get(name) - if bundle is None: - raise InstallerError(f"harness-index.yaml lists bundle '{name}', but no files for it " - "were shipped") - if digest != bundle.digest: - raise InstallerError( - f"harness bundle '{name}': the chart computed {digest}, the installer " - f"{bundle.digest}; the files changed on the way or the tree_digest contract " - "differs between templates/configmap-bom.yaml and apply_bom.py") - for name in sorted(set(bundles) - set(listed)): - if listed or index: - raise InstallerError(f"harness bundle '{name}' is missing from harness-index.yaml") def agent_mounts_planned(profiles): @@ -1610,138 +949,6 @@ def validate_harness(profiles, bundles): return revisions -def validate_profiles(profiles, known_profiles=(), governed=False): - """Catch mistakes before anything touches the gateway. - - governed: the gateway takes its provider profiles from the governance - interceptor, which may serve profiles the installer does not ship - (overrides/governance-profiles). A type that is neither built-in nor - shipped is then checked against the live catalog at apply time - (ProfileApplier.adopt_governed_profiles) instead of refused here.""" - errors = [] - seen = {} - for profile, ws in enabled_workspaces(profiles): - where = f"{profile.name}/{ws.name}" - if not NAME_RE.match(ws.name) or len(ws.name) > OPENSHELL_NAME_LIMIT: - errors.append(f"{where}: workspace name must be a DNS label of at most {OPENSHELL_NAME_LIMIT} characters") - if ws.name in seen: - errors.append(f"{where}: workspace '{ws.name}' is also defined by profile {seen[ws.name]}") - seen[ws.name] = profile.name - provider_names = [p.name for p in ws.providers if p.enabled] - provider_types = {p.name: p.type for p in ws.providers if p.enabled} - if len(provider_names) != len(set(provider_names)): - errors.append(f"{where}: duplicate provider names") - for p in ws.providers: - if not p.enabled: - continue - if not isinstance(p.credential_free, bool): - errors.append(f"{where}: provider '{p.name}' credentialFree must be a boolean") - elif p.credential_free: - # A keyless provider is an attachment to a governed service - # (an in-cluster MCP Service): no key, no model, and a type - # that is a live governance profile, not a built-in type. - for label, value in (("name", p.name), ("type", p.type)): - if not NAME_RE.match(value or "") or len(value) > OPENSHELL_NAME_LIMIT: - errors.append(f"{where}: keyless provider {label} must be a " - f"DNS label of at most {OPENSHELL_NAME_LIMIT} characters") - if p.type in PROVIDER_CRED_MAP: - errors.append(f"{where}: provider '{p.name}' credentialFree cannot disable " - f"the credential requirement of built-in type '{p.type}'") - if p.credential_secret: - errors.append(f"{where}: provider '{p.name}' credentialFree cannot be " - "combined with credentialSecret") - if (p.model or p.nemoclaw_provider or p.model_secret_key or p.base_url_secret_key - or p.base_url or p.inference_timeout): - errors.append(f"{where}: keyless provider '{p.name}' cannot be combined " - "with model/inference settings") - if governed and p.name != p.type: - # OpenShell 0.1.2's governance interceptor validates a new - # sandbox's providers by comparing each provider *name* with - # the vended profile ids (examples/governance-interceptor, - # providers_are_managed): a provider named differently from - # its type is refused at sandbox create with "sandbox - # providers may only use vended provider profiles". Found - # live; refuse it here, with the reason, instead. - errors.append(f"{where}: provider '{p.name}' must be named after its type " - f"'{p.type}' (the governance interceptor checks sandbox providers " - "by profile id, so name and type must match)") - if (not p.credential_free and p.type not in PROVIDER_CRED_MAP - and p.type not in known_profiles): - if not governed: - errors.append(f"{where}: provider '{p.name}' has unsupported type '{p.type}' " - "(no built-in type and no shipped provider profile)") - elif not NAME_RE.match(p.type or "") or len(p.type) > OPENSHELL_NAME_LIMIT: - errors.append(f"{where}: provider '{p.name}' type must be a DNS label of at " - f"most {OPENSHELL_NAME_LIMIT} characters (a governance profile id)") - if not p.credential_secret and not p.credential_free: - errors.append(f"{where}: provider '{p.name}' has no credentialSecret") - elif p.credential_secret and not SECRET_NAME_RE.match(p.credential_secret): - errors.append(f"{where}: provider '{p.name}' has an invalid credentialSecret name") - if not SECRET_KEY_RE.match(p.credential_secret_key or ""): - errors.append(f"{where}: provider '{p.name}' has an invalid credentialSecretKey") - for field_name, value in (("baseUrlSecretKey", p.base_url_secret_key), - ("modelSecretKey", p.model_secret_key)): - if value and not SECRET_KEY_RE.match(value): - errors.append(f"{where}: provider '{p.name}' has an invalid {field_name}") - if p.base_url_secret_key and p.type not in BASE_URL_CONFIG_KEYS: - errors.append(f"{where}: provider '{p.name}' of type '{p.type}' does not take a base URL " - f"(supported: {', '.join(sorted(BASE_URL_CONFIG_KEYS))})") - if p.inference_timeout < 0: - errors.append(f"{where}: provider '{p.name}' has a negative inferenceTimeout") - if p.refresh_strategy and p.refresh_strategy != "oauth2_refresh_token": - errors.append(f"{where}: provider '{p.name}' has unsupported refresh strategy '{p.refresh_strategy}'") - if p.credentials_spec: - if not isinstance(p.credentials_spec, dict): - errors.append(f"{where}: provider '{p.name}' credentials must be a mapping of " - "the profile's credential name to {secretKey[, title]}") - elif p.credential_free: - errors.append(f"{where}: keyless provider '{p.name}' cannot have credentials") - else: - for cname, entry in p.credentials_spec.items(): - if not CREDENTIAL_NAME_RE.match(str(cname)): - errors.append(f"{where}: provider '{p.name}' credential '{cname}' is " - "not a valid credential name") - if not isinstance(entry, dict) or not SECRET_KEY_RE.match( - str(entry.get("secretKey", ""))): - errors.append(f"{where}: provider '{p.name}' credential '{cname}' " - "needs a valid secretKey") - elif set(entry) - {"secretKey", "title"}: - errors.append(f"{where}: provider '{p.name}' credential '{cname}' has " - "unknown keys " - f"{', '.join(sorted(set(entry) - {'secretKey', 'title'}))}") - sandbox_names = set() - for s in ws.sandboxes: - if not s.enabled: - continue - if s.name in sandbox_names: - errors.append(f"{where}: duplicate sandbox '{s.name}'") - sandbox_names.add(s.name) - if len(s.name) > OPENSHELL_NAME_LIMIT or not NAME_RE.match(s.name): - errors.append(f"{where}: sandbox name '{s.name}' must be a DNS label of at most {OPENSHELL_NAME_LIMIT} characters") - if s.type not in SANDBOX_TYPES: - errors.append(f"{where}: sandbox '{s.name}' has unsupported type '{s.type}'") - for ref in s.providers: - if ref not in provider_names: - errors.append(f"{where}: sandbox '{s.name}' uses provider '{ref}' which is not an enabled provider in this workspace") - if s.type in ("openclaw", "nemoclaw") and not provider_names: - errors.append(f"{where}: {s.type} sandbox '{s.name}' needs at least one provider") - if s.inference: - errors += [f"{where}: sandbox '{s.name}' {p}" - for p in sandbox_inference_problems( - s, provider_types, {p.name: p.type for p in ws.providers})] - if not isinstance(s.env, dict): - errors.append(f"{where}: sandbox '{s.name}' env must be a mapping of NAME to " - "{value} or {secret, secretKey}") - else: - for name, src in s.env.items(): - errors += [f"{where}: sandbox '{s.name}' env '{name}' {problem}" - for problem in sandbox_env_problems(name, src)] - if s.gpu_enabled and s.gpu_count < 1: - errors.append(f"{where}: sandbox '{s.name}' gpu.count must be at least 1") - if errors: - raise InstallerError("invalid profiles:\n - " + "\n - ".join(errors)) - - def check_profiles_against_bom(profiles, bom): """Software a profile needs must be in the BOM.""" if "nemoclaw" in bom["spec"]: @@ -1936,9 +1143,9 @@ def _resolve_proxy(profiles, secrets_dir, proxy): "is present on a VM disk; provider keys belong only to " "the credential proxy") check_binding_provider(p, configured, bindings[p.credential_secret, skey]) - p.credential_values[cname] = f"{prefix}.{p.credential_secret}.{skey}" - creds.setdefault(ws.name, {})[p.name] = ( - f"{prefix}.{p.credential_secret}.{p.credential_secret_key}") + p.credential_values[cname] = _saw_core.credential_marker(prefix, p.credential_secret, skey) + creds.setdefault(ws.name, {})[p.name] = _saw_core.credential_marker( + prefix, p.credential_secret, p.credential_secret_key) return creds @@ -1951,99 +1158,6 @@ def read_secret_value(base, key): # Characters a base URL never needs and a shell would interpret; the URL ends # up in an OpenClaw command line inside the sandbox. -SHELL_UNSAFE = set("\"'`$\\;|&<>(){}") - -# Environment variable names (a credential's `env`, a sandbox `env:` key); -# OpenShell's own rule for `sandbox create --env`. -ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -# A profile credential name (`credentials[].name`). -CREDENTIAL_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") -# OpenShell's limit for one `--env` value. -ENV_VALUE_LIMIT = 8192 -# A value read from a Secret, as Kubernetes spells it: -# secretKeyRef: {name: , key: } -# with an optional default (used when the key is absent), required (defaults -# to "no default given") and title (the RHDH form's label). -SANDBOX_ENV_KEYS = {"value", "title"} -INPUT_KEYS = {"secretKeyRef", "default", "required", "title"} - - -def secret_ref(src): - """(name, key) of a {secretKeyRef: {name, key}} input, or None.""" - ref = src.get("secretKeyRef") if isinstance(src, dict) else None - if not isinstance(ref, dict): - return None - return str(ref.get("name", "")), str(ref.get("key", "")) - - -def secret_ref_problems(src): - ref = secret_ref(src) - if ref is None: - return ["needs secretKeyRef: {name, key}"] - out = [] - if not SECRET_NAME_RE.match(ref[0]): - out.append("secretKeyRef.name must be a valid Secret name") - if not SECRET_KEY_RE.match(ref[1]): - out.append("secretKeyRef.key must be a valid Secret key") - if set(src["secretKeyRef"]) - {"name", "key"}: - out.append("secretKeyRef takes only name and key") - return out -INFERENCE_KEYS = {"providerRef", "protocol", "baseUrl", "model", "models", "timeoutSeconds"} -# The API protocols the installer can onboard an agent with. A provider's -# *profile* (type) says which hosts a credential may reach; the protocol says -# how the agent talks to them, so a governed profile for another host (e.g. -# vllm-prod) works with protocol openai. -INFERENCE_PROTOCOLS = {"openai", "anthropic"} -# The protocol a built-in provider type speaks (the legacy path, no -# `inference:` block, and the default when a sandbox names a provider of -# such a type). -PROVIDER_PROTOCOLS = {"openai": "openai", "codex": "openai", "nvidia": "openai", - "build": "openai", "anthropic": "anthropic", "claude-code": "anthropic"} - - -def check_env_value(value): - """A value OpenShell accepts for `--env`: one line, at most 8 KiB. The - value is not echoed (it may come from a Secret).""" - if "\n" in value or "\r" in value or "\0" in value: - raise ValueError("value must be a single line") - if len(value.encode()) > ENV_VALUE_LIMIT: - raise ValueError(f"value is longer than {ENV_VALUE_LIMIT} bytes") - return value - - -def sandbox_env_problems(name, src): - """What is wrong with one `env:` entry of a sandbox ([] when fine).""" - out = [] - if not ENV_NAME_RE.match(str(name)): - out.append("is not a valid variable name ([A-Za-z_][A-Za-z0-9_]*)") - elif str(name).startswith("OPENSHELL_"): - out.append("is reserved by OpenShell (OPENSHELL_*)") - elif name in set(PROVIDER_CRED_MAP.values()): - out.append("is a provider credential variable; credentials go on a provider, " - "the agent can read sandbox env") - elif name in set(BASE_URL_CONFIG_KEYS.values()) or name in ("MODEL", "OPENCLAW_MODEL", - "NEMOCLAW_MODEL"): - out.append("is an inference setting; set it in the sandbox's inference: block") - if isinstance(src, dict) and ("secretKeyRef" in src or "default" in src): - # A value from a Secret would travel in `sandbox create --env`'s argv - # and be readable by every process in the sandbox (review, P1): - # secrets go on a provider, whose profile the gateway injects. - return out + ["must be a literal ({value: ...}); a value from a Secret belongs on a " - "provider, not in sandbox env (the agent and argv could read it)"] - if not isinstance(src, dict) or "value" not in src: - return out + ["needs {value: ...}"] - unknown = sorted(set(src) - SANDBOX_ENV_KEYS) - if unknown: - out.append(f"has unknown keys {', '.join(unknown)}") - if isinstance(src["value"], bool) or not isinstance(src["value"], (str, int, float)): - out.append("value must be a string (quote it)") - else: - try: - check_env_value(str(src["value"])) - except ValueError as exc: - out.append(f"value: {exc}") - return out - def same_sandbox(seen, recorded): """Is the running sandbox the one the record describes? @@ -2083,128 +1197,17 @@ def env_flags(values): for part in ("--env", f"{name}={value}")] -def input_problems(src, kinds=(str,)): - """What is wrong with one typed input (a literal or {secret, secretKey, - default?, required?, title?}); [] when fine.""" - if not isinstance(src, dict): - return [] if isinstance(src, kinds) and not isinstance(src, bool) else [ - "must be a string or {secretKeyRef: {name, key}, ...}"] - out = [] - unknown = sorted(set(src) - INPUT_KEYS) - if unknown: - out.append(f"has unknown keys {', '.join(unknown)}") - out += secret_ref_problems(src) - if "required" in src and not isinstance(src["required"], bool): - out.append("required must be true or false") - if "default" in src and (isinstance(src["default"], bool) - or not isinstance(src["default"], kinds)): - out.append("default must be a string") - return out - - def resolve_input(src, secrets_dir, what, forbidden=()): - """A typed input's value: the literal, or the Secret key it refers to - (secretKeyRef), else its default; "" when optional and absent.""" - if not isinstance(src, dict): - return str(src) - sname, skey = secret_ref(src) - if (sname, skey) in forbidden: - raise InstallerError(f"{what}: secretKeyRef names a provider credential or refresh material; refusing to expose it") - value = read_secret_value(Path(secrets_dir) / sname, skey) - if value: - return value - if "default" in src: - return str(src["default"]) - if src.get("required", "default" not in src): - raise InstallerError(f"{what} not found: Secret '{sname}' key '{skey}'. List the " - "Secret in openshell-saw additionalProviderSecrets and make sure " - "it exists.") - return "" - - -def inference_protocol(spec, provider_type): - """The protocol an `inference:` block uses: its own `protocol`, else the - one the provider's type implies ("" when neither says).""" - return spec.get("protocol") or PROVIDER_PROTOCOLS.get(provider_type or "", "") - - -def sandbox_inference_problems(sb, provider_types, declared_provider_types=None): - """What is wrong with a sandbox's `inference:` block ([] when fine). - `provider_types` maps the workspace's enabled provider names to their - types.""" - spec = sb.inference - provider_names = provider_types - if not isinstance(spec, dict): - return ["inference must be a mapping"] - out = [] - unknown = sorted(set(spec) - INFERENCE_KEYS) - if unknown: - out.append(f"inference has unknown keys {', '.join(unknown)}") - ref = spec.get("providerRef") - if not ref: - out.append("inference needs providerRef (the provider whose credential it uses)") - elif ref not in provider_names: - out.append(f"inference providerRef '{ref}' is not an enabled provider in this workspace") - elif ref not in sb.providers: - out.append(f"inference providerRef '{ref}' is not in the sandbox's providers") - protocol = inference_protocol(spec, provider_types.get(ref or "")) - if "protocol" in spec and spec["protocol"] not in INFERENCE_PROTOCOLS: - out.append(f"inference protocol must be one of {', '.join(sorted(INFERENCE_PROTOCOLS))}") - elif not protocol and ref in provider_names: - out.append(f"inference needs protocol ({' | '.join(sorted(INFERENCE_PROTOCOLS))}): " - f"provider type '{provider_types[ref]}' does not imply one") - if "model" not in spec: - out.append("inference needs model") - model = spec.get("model") - if model == "" or (isinstance(model, dict) and not model.get("default") - and not model.get("required", "default" not in model)): - out.append("inference model must be non-empty: use a non-empty default or required: true") - for key in ("baseUrl", "model"): - if key in spec: - out += [f"inference {key} {p}" for p in input_problems(spec[key])] - models = spec.get("models", {}) - if not isinstance(models, dict): - out.append("inference models must be a mapping of provider names to model inputs") - else: - declared = declared_provider_types if declared_provider_types is not None else provider_types - if models and sb.type != "openclaw": - out.append("inference models is supported only for openclaw sandboxes") - for name, model in models.items(): - if name not in declared: - out.append(f"inference models names unknown provider '{name}'") - elif declared[name] not in ("openai", "anthropic"): - out.append(f"inference models '{name}' needs a native OpenAI or Anthropic provider") - out += [f"inference models '{name}' {p}" for p in input_problems(model)] - if model == "" or (isinstance(model, dict) and not model.get("default") - and not model.get("required", "default" not in model)): - out.append(f"inference models '{name}' must be non-empty") - if "timeoutSeconds" in spec: - out += [f"inference timeoutSeconds {p}" - for p in input_problems(spec["timeoutSeconds"], (str, int))] - return out + return resolve_bom_input(src, lambda name, key: read_secret_value(Path(secrets_dir) / name, key), + what, forbidden) def resolve_sandbox_inference(ws, sb, secrets_dir, mode="proxy", forbidden=()): - """A sandbox's `inference:` resolved: providerRef, protocol, baseUrl - (validated, "" = the protocol's native API), model, timeoutSeconds.""" - spec = sb.inference - where = f"sandbox '{sb.name}' in workspace '{ws.name}'" - provider = next((p for p in ws.providers if p.enabled and p.name == spec["providerRef"]), None) - out = {"providerRef": spec["providerRef"], - "protocol": inference_protocol(spec, provider.type if provider else "")} - out["model"] = resolve_input(spec["model"], secrets_dir, f"{where}: inference model", forbidden) - if not out["model"]: - raise InstallerError(f"{where}: inference model is empty") - url = resolve_input(spec["baseUrl"], secrets_dir, f"{where}: inference baseUrl", forbidden) \ - if "baseUrl" in spec else "" - if url: - try: - url = check_base_url(url) - except ValueError as exc: - raise InstallerError(f"{where}: inference baseUrl: {exc}") from None - if mode == "proxy" and provider and not provider.credential_free: + def validate_endpoint(url, spec, provider, where): + if mode != "proxy" or not provider or provider.credential_free: + return + if url: check_proxy_endpoint(url, f"{where}: inference baseUrl") - if mode == "proxy" and provider and not provider.credential_free: installer = Path(__file__).resolve().parent profile_type = "nvidia" if provider.type == "build" else provider.type shipped = (profile_type in provider_profiles(installer) or @@ -2214,50 +1217,10 @@ def resolve_sandbox_inference(ws, sb, secrets_dir, mode="proxy", forbidden=()): "proxy host profile; inference baseUrl must use secretKeyRef " f"{{name: {provider.credential_secret}, key: url}} so the proxy " "can bind the credential to the Secret's url host") - out["baseUrl"] = url - timeout = resolve_input(spec["timeoutSeconds"], secrets_dir, - f"{where}: inference timeoutSeconds", forbidden) if "timeoutSeconds" in spec else "" - if timeout and (not str(timeout).isdigit() or int(timeout) < 0): - raise InstallerError(f"{where}: inference timeoutSeconds must be a non-negative integer") - out["timeoutSeconds"] = int(timeout) if timeout else 0 - if "models" in spec: - out["models"] = {} - for name, src in spec["models"].items(): - attached = next((p for p in ws.providers if p.enabled and p.name == name - and name in sb.providers), None) - if attached is None: - continue - if attached.base_url or attached.base_url_secret_key: - raise InstallerError(f"{where}: inference models '{name}' needs a native API") - model = resolve_input(src, secrets_dir, f"{where}: inference models '{name}'", forbidden) - if not model: - raise InstallerError(f"{where}: inference models '{name}' is empty") - try: - check_env_value(model) - except ValueError as exc: - raise InstallerError(f"{where}: inference models '{name}': {exc}") from None - out["models"][name] = model - for key in ("model", "baseUrl"): - try: - check_env_value(out[key]) - except ValueError as exc: - raise InstallerError(f"{where}: inference {key}: {exc}") from None - return out - - -def sandbox_protocol(sb, provider): - """How the agent talks to its inference endpoint.""" - if sb.inference_values: - return sb.inference_values["protocol"] - return PROVIDER_PROTOCOLS.get(provider.type, "") if provider else "" - -def sandbox_model(sb, provider, default=""): - """The model the agent runs: `inference.model`, else the legacy sandbox - `model`, the provider's `model`/`modelSecretKey`, the built-in default.""" - if sb.inference_values: - return sb.inference_values["model"] - return sb.model or (provider.model if provider else "") or default + def read_value(name, key): + return read_secret_value(Path(secrets_dir) / name, key) + return resolve_inference(ws, sb, read_value, validate_endpoint, forbidden) def openclaw_agent_model(model_ref, current, preserve_binding=True): @@ -2271,37 +1234,6 @@ def openclaw_agent_model(model_ref, current, preserve_binding=True): return {**current, "primary": model_ref} if isinstance(current, dict) else model_ref -def sandbox_endpoint(sb, provider): - """The endpoint the agent is onboarded against: `inference.baseUrl`, else - the provider's own base URL ("" = the protocol's native API).""" - if sb.inference_values: - return sb.inference_values["baseUrl"] - return provider.base_url if provider else "" - - -def check_base_url(url): - """An http(s) base URL without credentials, query or fragment. The value - is not echoed in errors: a pasted URL may contain a token.""" - try: - parts = urlsplit(url) - ok = (parts.scheme in ("http", "https") and parts.hostname - and parts.username is None and parts.password is None - and not parts.query and not parts.fragment - and not any(c.isspace() or c in SHELL_UNSAFE for c in url)) - if ok: - parts.port # raises ValueError on a bad port - except ValueError: - ok = False - if not ok: - raise ValueError("base URL must be http(s)://host[:port][/path] without credentials, " - "query or fragment") - host = parts.hostname - if host in ("localhost", "127.0.0.1", "::1"): - raise ValueError("base URL must not be localhost: inside the sandbox that is the " - "sandbox itself (use a cluster Service or Route host)") - return url.rstrip("/") - - # --------------------------------------------------------------------------- # State (what is installed) # --------------------------------------------------------------------------- @@ -3252,29 +2184,6 @@ def ws_args(name): return [] if name == "default" else ["--workspace", name] -_TOOL_GATE_PACKAGE = """\ -{ - "name": "saw-tool-gate", - "version": "0.1.0", - "private": true, - "type": "module", - "openclaw": { - "extensions": ["./index.js"], - "runtimeExtensions": ["./index.js"] - } -} -""" - -_TOOL_GATE_MANIFEST = """\ -{ - "id": "saw-tool-gate", - "name": "SAW tool gate", - "description": "Audit every tool call with the signed-in user and current task, and require approval before consequential tools run.", - "activation": {"onStartup": true}, - "configSchema": {"type": "object", "additionalProperties": false} -} -""" - # CJS so `node -e` can update OpenClaw's writable configuration. All trusted # code, policy and identity are already on the read-only volume. _TOOL_GATE_CONFIG = """\ @@ -3613,6 +2522,38 @@ def verify(self, volume, workspace, source, expected_digest): return [] +class _VMWorkspaceSequence: + """VM effects behind the shared workspace/provider/sandbox ordering.""" + + def __init__(self, applier): + self.applier = applier + + def prepare_workspace(self, workspace): + self.applier.apply_workspace(workspace) + self.applier.adopt_governed_profiles(workspace) + self.applier.prepare_credential_migration(workspace) + return True + + def apply_provider(self, workspace, provider): + self.applier.apply_provider(workspace, provider) + return True + + def apply_sandbox(self, workspace, sandbox): + if not sandbox.enabled: + log(f"Sandbox '{sandbox.name}' disabled, skipping") + return True + try: + self.applier.apply_sandbox(workspace, sandbox) + return True + except InstallerError as exc: + log(f"ERROR: sandbox '{sandbox.name}': {exc}") + self.applier.sandbox_failures.append(f"sandbox '{sandbox.name}': {exc}") + except Exception as exc: + log(f"ERROR: sandbox '{sandbox.name}': {exc}\n{traceback.format_exc()}") + self.applier.sandbox_failures.append(f"sandbox '{sandbox.name}': {exc}") + return False + + class ProfileApplier: def __init__(self, shell, cfg, creds, provider_profile_docs=None, harness=None): self.sh = shell @@ -4117,13 +3058,8 @@ def find_provider(self, ws, names, sb=None): preferring one with a model. Never another provider of the workspace: found live, a skipped `custom` provider made OpenClaw onboard with `brave` and a default NVIDIA model.""" - usable = [p for p in self.usable(ws) if not p.credential_free] - if sb is not None and sb.inference_values: - ref = sb.inference_values["providerRef"] - return next((p for p in usable if p.name == ref), None) - if names: - usable = [p for n in names for p in usable if p.name == n] - return next((p for p in usable if p.model), usable[0] if usable else None) + ref = sb.inference_values["providerRef"] if sb is not None and sb.inference_values else None + return select_inference_provider(self.usable(ws), names, inference_ref=ref) def sandbox_state(self, ws, sb): """'running', 'broken' (Error/Completed), 'deleting' or 'missing'.""" @@ -4199,37 +3135,10 @@ def desired_harness_mount(self, ws, sb): def tool_gate_identity(self, ws, sb): dashboard = self.cfg.get("dashboard") or {} owner = dashboard.get("owner") if isinstance(dashboard, dict) else "" - subject = self.cfg.get("ownerSubject") or "" - if not isinstance(owner, str) or not owner.strip(): - raise InstallerError(f"sandbox '{sb.name}': dashboard.owner is required for the tool gate") - if not isinstance(subject, str) or not subject.strip(): - raise InstallerError(f"sandbox '{sb.name}': ownerSubject is required for the tool gate") - return {"username": owner, "subject": subject, - "sandbox": sb.name, "workspace": ws.name} + return _saw_core.tool_gate_identity(owner, self.cfg.get("ownerSubject"), ws.name, sb.name) def tool_gate_bundle(self, ws, sb): - """Derive the trusted tree and expected digest without mutating a volume.""" - identity = self.tool_gate_identity(ws, sb) - here = Path(__file__).resolve().parent - names = ("tool-actions.yaml", "tool-gate.mjs", "tool-gate-plugin.mjs") - missing = [name for name in names if not (here / name).is_file()] - if missing: - raise InstallerError("tool gate files missing from the installer disk: " + ", ".join(missing)) - policy = (here / "tool-actions.yaml").read_bytes() - policy_hash = "sha256:" + hashlib.sha256(policy).hexdigest() - plugin = (here / "tool-gate-plugin.mjs").read_bytes().replace( - b"__SAW_POLICY_HASH__", policy_hash.encode()) - tree = { - "tool-actions.yaml": (policy, False), - "identity.json": ((json.dumps(identity, sort_keys=True) + "\n").encode(), False), - "plugin/package.json": (_TOOL_GATE_PACKAGE.encode(), False), - "plugin/openclaw.plugin.json": (_TOOL_GATE_MANIFEST.encode(), False), - "plugin/index.js": (plugin, False), - "plugin/tool-gate.mjs": ((here / "tool-gate.mjs").read_bytes(), False), - } - digest = harness_tree_digest(tree) - source = f"gate:{digest}" - return tree, source, digest, policy_hash + return _saw_core.tool_gate_bundle(Path(__file__).resolve().parent, self.tool_gate_identity(ws, sb)) def prepare_tool_gate(self, ws, sb, existing=False): """Fill trusted plugin, policy and identity from the host, before create.""" @@ -4524,27 +3433,7 @@ def check_harness_governance(self, ws, sb, info): return catalog = self.governance_catalog(ws) attached = {p.type for p in self.usable(ws) if p.name in sb.providers} - for item in info["governance"]: - profile = item["governanceProfile"] - what = f"harness '{info['name']}' {item['kind']} '{item['name']}'" - if profile not in catalog: - raise InstallerError( - f"{what} names governanceProfile '{profile}', which the gateway does not " - f"serve in workspace '{ws.name}'; refusing to fill the harness of sandbox " - f"'{sb.name}'") - if profile not in attached: - raise InstallerError( - f"{what} names governanceProfile '{profile}', but sandbox '{sb.name}' has no " - f"provider of type '{profile}'; add one to the workspace's providers.yaml and " - "to the sandbox's providers (its endpoints and key reach the sandbox only " - "through a provider)") - for host in item["hosts"]: - if not host_allowed(host, catalog[profile]): - allowed = ", ".join(sorted(catalog[profile])) or "none" - raise InstallerError( - f"{what} reaches {host}, which governanceProfile '{profile}' does not " - f"allow (allowed: {allowed}); refusing to fill the harness of " - f"sandbox '{sb.name}'") + _saw_core.check_harness_governance(ws, sb, info, catalog, attached) def expected_harness_digest(self, volume, bundle, source): """Trusted tree digest for `volume`, or None when none is known yet. @@ -5005,7 +3894,8 @@ def start_openclaw(self, ws, sb, provider): # No /sandbox chown: OpenShell 0.1.x runs the workload without # capabilities (root in the container cannot even read /sandbox) and # already gives /sandbox to the image's user. - model = sandbox_model(sb, provider, "nvidia/nemotron-3.5-lightning-30b-a3b") + inference = openclaw_inference_settings(sb, provider) + model = inference["model"] oc_env = OPENCLAW_EXEC_ENV # OpenShell 0.1.x: no inference.local. OpenClaw calls the provider's # own endpoint with the placeholder key this exec receives in the @@ -5016,22 +3906,9 @@ def start_openclaw(self, ws, sb, provider): # (`inference:`), else the provider type's (legacy). The provider # only names the credential: its profile decides which hosts the # key may reach; the protocol decides how the agent talks to them. - endpoint = sandbox_endpoint(sb, provider) - protocol = sandbox_protocol(sb, provider) - if sb.inference_values: - # No baseUrl: the provider type's own API (NVIDIA's, say), else - # the protocol's official one. An official API (openai, - # anthropic) is onboarded natively (OpenClaw's own catalog) when - # the provider is that API; anything else as a custom endpoint. - base_url = (endpoint or NATIVE_BASE_URLS.get(provider.type, "") - or PROTOCOL_NATIVE_BASE_URLS.get(protocol, "")) - native = (not endpoint and provider.type in OPENCLAW_AUTH_CHOICES - and PROVIDER_PROTOCOLS.get(provider.type) == protocol) - else: - base_url = endpoint or NATIVE_BASE_URLS.get(provider.type, "") - native = not (endpoint or provider.base_url_secret_key) + base_url, protocol = inference["base_url"], inference["protocol"] key_var = credential_env_name(provider.type, self.provider_profiles) - auth_choice = OPENCLAW_AUTH_CHOICES.get(provider.type) if native else None + auth_choice = inference["auth_choice"] if (not base_url and not auth_choice) or not key_var: log(f"WARN: no endpoint for provider '{provider.name}' (type '{provider.type}'); " f"set the sandbox's inference.baseUrl. Skipping OpenClaw onboarding") @@ -5224,8 +4101,7 @@ def configure_harness(self, ws, sb, exec_cmd, oc_env): ids = harness_plugin_ids(info) if info is not None else [] if ids: script = "; ".join( - f"{oc_env} openclaw plugins enable {shlex.quote(i)} --accept-capabilities " - f"|| {oc_env} openclaw plugins enable {shlex.quote(i)}" for i in ids) + _saw_core.enable_harness_plugin(i, oc_env) for i in ids) self.cli(*exec_cmd, "sh", "-c", script, check=False) self.configure_harness_mcp(ws, sb, exec_cmd, oc_env, info) @@ -5600,24 +4476,8 @@ def apply(self, profiles): for profile, ws in enabled_workspaces(profiles): self.profile_name = profile.name banner(f"Profile {profile.name} / workspace {ws.name}") - self.apply_workspace(ws) - self.adopt_governed_profiles(ws) - self.prepare_credential_migration(ws) - for provider in ws.providers: - if provider.enabled: - self.apply_provider(ws, provider) - for sb in ws.sandboxes: - if sb.enabled: - try: - self.apply_sandbox(ws, sb) - except InstallerError as exc: - log(f"ERROR: sandbox '{sb.name}': {exc}") - self.sandbox_failures.append(f"sandbox '{sb.name}': {exc}") - except Exception as exc: - log(f"ERROR: sandbox '{sb.name}': {exc}\n{traceback.format_exc()}") - self.sandbox_failures.append(f"sandbox '{sb.name}': {exc}") - else: - log(f"Sandbox '{sb.name}' disabled, skipping") + apply_workspace_sequence(ws, (p for p in ws.providers if p.enabled), + ws.sandboxes, _VMWorkspaceSequence(self)) self.finish_prune() self.cleanup_harness_volumes() self.cleanup_harness_images() @@ -5893,7 +4753,9 @@ def restore_sandbox(self, ws, sb, record): # directory metadata; then overwrite writable files in place # (their parent directory may belong to the image's root). "readonly=$(mktemp) || exit 1; trap 'rm -f \"$readonly\"' EXIT; " - "find . ! -type d ! -writable -printf '%p\\n%P\\n' > \"$readonly\" || exit 1; " + "find . ! -type d -exec sh -c 'for path do " + "if [ ! -w \"$path\" ]; then printf \"%s\\n\" \"$path\" \"${path#./}\" || exit 1; " + "fi; done' sh {} + > \"$readonly\" || exit 1; " f"tar -xzf {self.PRESERVE_ARCHIVE} --skip-old-files " "--same-permissions --no-same-owner || exit 1; " f"LC_ALL=C tar -tzf {self.PRESERVE_ARCHIVE} | grep -v '/$' | " @@ -6470,323 +5332,7 @@ def openclaw_gateway_script(cfg, workspace, sandbox, oc_env, refilled=False, pro # At most this many connections reach `openshell forward service` per UI # (OpenShell allows 20 per sandbox; the rest are left for exec sessions). FORWARD_MAX_CONNECTIONS = 16 -SANDBOX_UI_LIMIT_PY = r'''"""Caps the connections to `openshell forward service` (SAW sandbox UI). - -OpenShell allows 20 concurrent forward connections per sandbox and closes -the rest (RESOURCE_EXHAUSTED, "sandbox SSH connection limit reached", -NVIDIA/OpenShell#3494). A browser loading the OpenClaw control UI through -the route opens more than that at once, and the requests that lost came -back as 502/504 after 30 s. This TCP relay sits between oauth2-proxy and the -forward: - -- at most MAX connections reach the forward; the others wait for a free one; -- while some wait, an HTTP keep-alive connection that has been idle for - IDLE_PREEMPT seconds after an answer is closed to make room (an HTTP - client opens a new one); WebSocket connections are never closed; -- a connection the forward closes or resets before answering is retried; -- a GET that gets no answer at all within ANSWER_WAIT is sent again on new - connections, and the first answer wins. - - python3 saw_ui_limit.py LISTEN_PORT UPSTREAM_PORT [MAX] -""" -import asyncio -import sys -import time - - -def log(message): - print(message, file=sys.stderr, flush=True) - -RETRIES = 8 -BUFFER_LIMIT = 1 << 20 -ANSWER_WAIT = 2.0 # seconds to wait for a first answer byte before giving up retries -IDLE_PREEMPT = 2.0 # seconds an answered keep-alive connection may hold a slot others wait for -HEDGES = 2 # extra connections for a repeatable request that gets no answer -HEDGE_TOTAL = 25.0 # seconds before giving up on such a request (the router allows 30) -# For a request that is not safe to send again and lost its answer. -_AMBIGUOUS_BODY = b"The sandbox closed the connection; the request may have been processed.\n" -AMBIGUOUS_ANSWER = (b"HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\n" - b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(_AMBIGUOUS_BODY)) + _AMBIGUOUS_BODY - - -class Conn: - """One client connection holding (or waiting for) a slot.""" - - def __init__(self, websocket): - self.websocket = websocket - self.last = time.monotonic() - self.answered = False # the forward spoke last: no request in flight - self.writers = [] - - def touch(self, answered): - self.last = time.monotonic() - self.answered = answered - - def idle_for(self): - return time.monotonic() - self.last - - def close(self): - for w in self.writers: - try: - w.transport.abort() - except Exception: - pass - - -class Slots: - """MAX upstream connections at a time; idle keep-alive ones give way. - - A client connection holds one slot for its upstream connection. A hedge - (an extra connection for a repeatable request that got no answer) takes - a slot of its own, only when one is free, and gives it back as soon as - it is closed or becomes the connection that answered.""" - - def __init__(self, limit): - self.limit = limit - self.active = set() - self.extra = 0 # hedge connections open now - self.cond = asyncio.Condition() - - def in_use(self): - return len(self.active) + self.extra - - def try_extra(self): - """Take a slot for a hedge connection if one is free; never waits. - (No await: atomic within the event loop.)""" - if self.in_use() >= self.limit: - return False - self.extra += 1 - return True - - async def release_extra(self, count=1): - if count <= 0: - return - async with self.cond: - self.extra = max(0, self.extra - count) - self.cond.notify(count) - - def victim(self): - idle = [c for c in self.active - if not c.websocket and c.answered and c.idle_for() >= IDLE_PREEMPT] - return min(idle, key=lambda c: c.last) if idle else None - - def describe(self): - ws = sum(1 for c in self.active if c.websocket) - busy = sum(1 for c in self.active if not c.websocket and not c.answered) - idle = sorted(round(c.idle_for(), 1) for c in self.active if not c.websocket and c.answered) - return (f"{self.in_use()}/{self.limit} in use: {ws} websocket, {busy} awaiting an answer, " - f"{len(idle)} answered (idle s: {idle}), {self.extra} hedge") - - async def acquire(self, conn): - start = time.monotonic() - logged = 0.0 - async with self.cond: - while self.in_use() >= self.limit: - v = self.victim() - if v is not None: - log(f"closing a keep-alive connection idle {v.idle_for():.1f}s to make room") - self.active.discard(v) - v.close() - break - waited = time.monotonic() - start - if waited - logged >= 5: - logged = waited - log(f"waiting {waited:.0f}s for a slot; {self.describe()}") - try: - await asyncio.wait_for(self.cond.wait(), 0.25) - except asyncio.TimeoutError: - pass - self.active.add(conn) - if time.monotonic() - start >= 1: - log(f"got a slot after {time.monotonic() - start:.1f}s; {self.describe()}") - - async def release(self, conn): - async with self.cond: - if conn in self.active: - self.active.discard(conn) - self.cond.notify() - - -async def relay(reader, writer, conn, answered): - """Copy until EOF, then half-close the other side.""" - try: - while True: - data = await reader.read(65536) - if not data: - break - conn.touch(answered) - writer.write(data) - await writer.drain() - if writer.can_write_eof(): - writer.write_eof() - except (ConnectionError, OSError): - pass - - -class Ambiguous(Exception): - """The request reached the forward, but no answer came back: it may or - may not have been processed, so it is not sent again.""" - - -async def open_answering(upstream, sent, slots): - """Connect to the forward and send the client's first bytes. - - Retries when nothing was sent (the connection failed), and, for a - request that is safe to send again (replayable), when the forward - closes or resets the connection without answering: that is how it - refuses a connection past its limit. Any other request may already - have been processed by then, so it is not sent again: Ambiguous.""" - for attempt in range(RETRIES + 1): - try: - ureader, uwriter = await asyncio.open_connection("127.0.0.1", upstream) - except OSError: - await asyncio.sleep(min(1.0, 0.2 * (attempt + 1))) - continue - try: - uwriter.write(sent) - await uwriter.drain() - # A refused forward connection closes (or resets) at once. A - # request still being sent (a large body) gets no answer yet: - # relay it. - first = await asyncio.wait_for(ureader.read(65536), ANSWER_WAIT) - except asyncio.TimeoutError: - if not hedgeable(sent): - log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); relaying without retries") - return ureader, uwriter, b"" - return await hedge(upstream, sent, ureader, uwriter, slots) - except (ConnectionError, OSError): - first = b"" - if first: - return ureader, uwriter, first - uwriter.close() - if not replayable(sent): - raise Ambiguous() - await asyncio.sleep(min(1.0, 0.2 * (attempt + 1))) - return None, None, b"" - - -def replayable(sent): - """A complete GET/HEAD/OPTIONS request without a body (a WebSocket - upgrade included: no answer means no upgrade): sending it again cannot - repeat a change.""" - head = sent.split(b"\r\n\r\n", 1) - return len(head) == 2 and not head[1] and sent.split(b" ", 1)[0] in (b"GET", b"HEAD", b"OPTIONS") - - -def hedgeable(sent): - """A replayable request that is not a WebSocket upgrade: safe to send - again on another connection while the first is still open.""" - return replayable(sent) and b"upgrade: websocket" not in sent.lower() - - -async def hedge(upstream, sent, ureader, uwriter, slots): - """Found live: a request through the forward sometimes got no answer at - all (no error, nothing in the forward's log) and the browser saw a 504. - For a request that is safe to repeat, send it again on new connections - and keep whichever answers first. Each extra connection needs a free - slot (Slots.try_extra); without one, the request just keeps waiting on - the connections it has.""" - pending = {asyncio.ensure_future(ureader.read(65536)): (ureader, uwriter)} - extras = 0 - deadline = time.monotonic() + HEDGE_TOTAL - try: - for extra in range(HEDGES + 1): - if extra < HEDGES: - if slots.try_extra(): - extras += 1 - log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); " - f"sending it again ({extra + 1})") - try: - r, w = await asyncio.open_connection("127.0.0.1", upstream) - w.write(sent) - await w.drain() - pending[asyncio.ensure_future(r.read(65536))] = (r, w) - except (ConnectionError, OSError): - pass - else: - log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); " - "no free slot to send it again") - wait = ANSWER_WAIT * 2 if extra < HEDGES else max(0.0, deadline - time.monotonic()) - done, _ = await asyncio.wait(pending, timeout=wait, return_when=asyncio.FIRST_COMPLETED) - for task in done: - r, w = pending.pop(task) - try: - data = task.result() - except (ConnectionError, OSError): - data = b"" - if data: - for other, (_, ow) in pending.items(): - other.cancel() - ow.close() - pending.clear() - return r, w, data - w.close() - if not pending: - break - for other, (_, ow) in pending.items(): - other.cancel() - ow.close() - log(f"no answer on {extras + 1} connections ({sent[:60]!r})") - return None, None, b"" - finally: - # The connection that answered (if any) runs on the client's own - # slot; every extra one taken here is closed by now. - await slots.release_extra(extras) - - -def handler(upstream, slots): - async def handle(creader, cwriter): - conn = None - try: - # The first bytes tell an HTTP request from a WebSocket upgrade, - # and are what a retry resends. - sent = await creader.read(65536) - if not sent: - return - conn = Conn(b"upgrade: websocket" in sent.lower()) - conn.writers.append(cwriter) - await slots.acquire(conn) - try: - ureader, uwriter, first = await open_answering(upstream, sent, slots) - except Ambiguous: - log(f"the forward closed without an answer ({sent[:60]!r}); " - "not sending it again, it may have been processed") - cwriter.write(AMBIGUOUS_ANSWER) - await cwriter.drain() - return - if ureader is None: - log("the forward refused the connection on every retry") - return - conn.writers.append(uwriter) - conn.touch(bool(first)) - if first: - cwriter.write(first) - await cwriter.drain() - await asyncio.gather(relay(creader, uwriter, conn, False), - relay(ureader, cwriter, conn, True)) - uwriter.close() - except (ConnectionError, OSError): - pass - finally: - if conn is not None: - await slots.release(conn) - cwriter.close() - return handle - - -async def main(listen, upstream, limit): - server = await asyncio.start_server(handler(upstream, Slots(limit)), "127.0.0.1", listen, - limit=BUFFER_LIMIT) - async with server: - await server.serve_forever() - - -if __name__ == "__main__": - listen_port, upstream_port = int(sys.argv[1]), int(sys.argv[2]) - max_conns = int(sys.argv[3]) if len(sys.argv) > 3 else 10 - asyncio.run(main(listen_port, upstream_port, max_conns)) -''' - +SANDBOX_UI_LIMIT_PY = _SAW_UI_LIMIT_SOURCE_BYTES.decode("utf-8") def sandbox_ui_units(cfg, home, cookie, gateway, issuer_ca=None): @@ -7246,91 +5792,42 @@ def _openssl(*args): @contextmanager def admin_session(cfg, home, owner, as_current_user=False): - """Temporarily trust one short-lived admin CA and keep its key in tmpfs.""" + """Use the common temporary identity with the VM's local trust adapter.""" gateway_tls = Path(home) / ".local/state/openshell/tls" - gateway_ca = gateway_tls / "ca.crt" - bundle = gateway_tls / "client-ca.pem" + gateway_ca, bundle = gateway_tls / "ca.crt", gateway_tls / "client-ca.pem" if not gateway_ca.is_file(): raise InstallerError("gateway CA certificate is missing") gateway_pem = gateway_ca.read_text(encoding="utf-8") uid = owner[0] if owner else os.getuid() runtime = Path(os.environ.get("SAW_RUNTIME_DIR", os.environ.get("XDG_RUNTIME_DIR", "/tmp"))) \ if as_current_user else Path(f"/run/user/{uid}") - if not runtime.is_dir(): - raise InstallerError(f"runtime tmpfs {runtime} is unavailable") - if not as_current_user: - fs = subprocess.run(["stat", "-f", "-c", "%T", str(runtime)], - capture_output=True, text=True, check=False) - if fs.returncode or fs.stdout.strip() != "tmpfs": - raise InstallerError(f"runtime directory {runtime} is not tmpfs") - root_base = runtime / "saw-root-admin" if as_current_user else Path("/run/saw/admin") - session_dir = root_dir = None - added = False + authority = runtime / "saw-root-admin" if as_current_user else Path("/run/saw/admin") try: - session_dir = Path(tempfile.mkdtemp(prefix="saw-admin.", dir=runtime)) - root_base.mkdir(mode=0o700, parents=True, exist_ok=True) - os.chmod(root_base, 0o700) - root_dir = Path(tempfile.mkdtemp(prefix="session.", dir=root_base)) - tls = session_dir / "tls" - client = tls / "client" - config_home = session_dir / "config" - client.mkdir(parents=True) - config_home.mkdir() - for directory in (session_dir, tls, client, config_home): - os.chmod(directory, 0o700) - if owner and os.geteuid() == 0: - os.chown(directory, *owner) - ca_crt, ca_key = root_dir / "ca.crt", root_dir / "ca.key" - _openssl("req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", - "-nodes", "-keyout", ca_key, "-out", ca_crt, - "-subj", "/O=openshell/OU=SAW-apply/CN=saw-apply-ca", "-days", "1", - "-addext", "basicConstraints=critical,CA:TRUE", - "-addext", "keyUsage=critical,keyCertSign") - csr, key, crt = root_dir / "client.csr", client / "tls.key", client / "tls.crt" - _openssl("req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", - "-nodes", "-keyout", key, "-subj", ADMIN_CERT_SUBJECT, "-out", csr) - ext = root_dir / "ext.cnf" - ext.write_text("basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\n" - "extendedKeyUsage=clientAuth\n", encoding="utf-8") - _openssl("x509", "-req", "-in", csr, "-CA", ca_crt, "-CAkey", ca_key, - "-set_serial", "0x" + secrets.token_hex(16), "-days", "1", - "-extfile", ext, "-out", crt) - shredded = subprocess.run(["shred", "-u", "--", str(ca_key)], - capture_output=True, check=False) - if shredded.returncode: - ca_key.unlink(missing_ok=True) - raise InstallerError("could not shred the per-run admin CA key") - shutil.copyfile(gateway_ca, tls / "ca.crt") - for file in (key, crt, tls / "ca.crt"): - os.chmod(file, 0o600) - if owner and os.geteuid() == 0: - os.chown(file, *owner) - admin_pem = ca_crt.read_text(encoding="utf-8") - write_if_changed(bundle, gateway_pem.rstrip() + "\n" + admin_pem, 0o600, owner) - added = True - if not as_current_user: - trust = ssl.create_default_context(cafile=str(gateway_ca)) - trust.load_cert_chain(str(crt), str(key)) - deadline = time.monotonic() + 30 - while True: - try: - with socket.create_connection(("127.0.0.1", GATEWAY_PORT), timeout=2) as sock: - with trust.wrap_socket(sock, server_hostname="host.openshell.internal"): - break - except (OSError, ssl.SSLError): - if time.monotonic() >= deadline: - raise InstallerError("gateway did not accept the per-run admin CA within 30 seconds") - time.sleep(1) - yield tls, config_home - finally: - try: - if added: - write_if_changed(bundle, gateway_pem, 0o600, owner) - finally: - if session_dir is not None: - shutil.rmtree(session_dir, ignore_errors=True) - if root_dir is not None: - shutil.rmtree(root_dir, ignore_errors=True) + with _saw_admin.identity(gateway_ca.read_bytes(), runtime, authority_base=authority, + subject=ADMIN_CERT_SUBJECT, owner=owner, + require_tmpfs=not as_current_user, crypto=_openssl) as identity: + added = False + try: + write_if_changed(bundle, gateway_pem.rstrip() + "\n" + identity["ca"], 0o600, owner) + added = True + if not as_current_user: + tls = identity["tls"] + trust = ssl.create_default_context(cafile=str(gateway_ca)) + trust.load_cert_chain(str(tls / "client/tls.crt"), str(tls / "client/tls.key")) + def accepted(): + try: + with socket.create_connection(("127.0.0.1", GATEWAY_PORT), timeout=2) as sock: + with trust.wrap_socket(sock, server_hostname="host.openshell.internal"): + return True + except (OSError, ssl.SSLError): + return False + _saw_admin.wait_for_trust(accepted, timeout=30) + yield identity["tls"], identity["config"] + finally: + if added: + write_if_changed(bundle, gateway_pem, 0o600, owner) + except _saw_admin.AdminSessionError as exc: + raise InstallerError(str(exc)) from None def cmd_install(args): @@ -7445,28 +5942,6 @@ def plan_for_user(cfg, profiles, creds, dashboard_script, in (harness.get("bundles") or {}).items()}}} -def harness_from_plan(data): - raw = data.get("harness") or {} - bundles = {} - for name, b in (raw.get("bundles") or {}).items(): - bundles[name] = HarnessBundle( - name=b["name"], agent=b["agent"], digest=b["digest"], files=b["files"]) - return {"bundles": bundles} - - -def profiles_from_plan(data): - profiles = [] - for p in data["profiles"]: - workspaces = [] - for w in p["workspaces"]: - workspaces.append(Workspace( - name=w["name"], enabled=w["enabled"], description=w["description"], - providers=[Provider(**x) for x in w["providers"]], - sandboxes=[Sandbox(**x) for x in w["sandboxes"]])) - profiles.append(Profile(name=p["name"], workspaces=workspaces)) - return profiles - - def cmd_apply(args): """Step 2 (root entry): apply the SAW-BOM profiles. @@ -7539,6 +6014,20 @@ def cmd_apply(args): script = copy_dir / "apply_bom.py" shutil.copyfile(Path(__file__).resolve(), script) os.chmod(script, 0o644) + if _SAW_CORE_EMBEDDED_B64 == "__SAW_CORE_" + "SOURCE_BASE64__": + # Checked-out source/test execution only. The rendered signed + # installer embeds the core and never reads an adjacent module. + core_copy = copy_dir / "saw_core.py" + core_copy.write_bytes(_SAW_CORE_SOURCE_BYTES) + os.chmod(core_copy, 0o644) + if _SAW_ADMIN_EMBEDDED_B64 == "__SAW_ADMIN_" + "SOURCE_BASE64__": + admin_copy = copy_dir / "saw_admin.py" + admin_copy.write_bytes(_SAW_ADMIN_SOURCE_BYTES) + os.chmod(admin_copy, 0o644) + if _SAW_UI_LIMIT_EMBEDDED_B64 == "__SAW_UI_LIMIT_" + "SOURCE_BASE64__": + transport_copy = copy_dir / "saw_ui_limit.py" + transport_copy.write_bytes(_SAW_UI_LIMIT_SOURCE_BYTES) + os.chmod(transport_copy, 0o644) # The runtime user executes this copy, so the tool-gate files it # reads from next to itself have to come along. They are part of # the installer disk (inputs.installer), not only the source tree. diff --git a/charts/openshell-saw/files/installer/saw_admin.py b/charts/openshell-saw/files/installer/saw_admin.py new file mode 100644 index 000000000..0a5a986fb --- /dev/null +++ b/charts/openshell-saw/files/installer/saw_admin.py @@ -0,0 +1,128 @@ +"""Temporary OpenShell administrator identity shared by both SAW backends. + +The caller owns gateway trust publication and serialization. Private material +exists only for this context; its CA key is destroyed immediately after signing. +The VM embeds this module in its signed installer, as it does saw_core.py. +""" +from contextlib import contextmanager +import os +from pathlib import Path +import secrets +import shutil +import subprocess +import tempfile +import time +import uuid + + +class AdminSessionError(RuntimeError): + pass + + +def require_memory(runtime): + probe = subprocess.run(["stat", "-f", "-c", "%T", str(runtime)], + capture_output=True, text=True, timeout=10) + if probe.returncode or probe.stdout.strip() != "tmpfs": + raise AdminSessionError("administrator session directory is not tmpfs") + + +def remove_abandoned(runtime): + """Caller must hold its session lock; emptyDir survives container restarts.""" + root = Path(runtime) + if (root / "saw-authorities").is_symlink(): + raise AdminSessionError("administrator authority path is a symlink") + for pattern in ("saw-admin.*", "saw-authorities/session.*"): + for path in root.glob(pattern): + if path.is_symlink(): + raise AdminSessionError("administrator session path is a symlink") + shutil.rmtree(path) + + +def openssl(*args): + try: + result = subprocess.run(["openssl", *map(str, args)], capture_output=True, timeout=20) + except (OSError, subprocess.SubprocessError): + raise AdminSessionError("administrator identity cryptographic tool unavailable") from None + if result.returncode: + raise AdminSessionError("administrator identity cryptographic operation failed") + return result.stdout + + +def erase_ca_key(path): + try: + result = subprocess.run(["shred", "-u", "--", str(path)], capture_output=True, timeout=20) + if result.returncode: + raise AdminSessionError("could not shred the per-run admin CA key") + finally: + path.unlink(missing_ok=True) + + +def issue(authority, client, *, subject, crypto=openssl, erase=erase_ca_key): + """Issue the VM's one-day P-256 CA/client pair, with the admin role in OU.""" + ca, ca_key = authority / "ca.crt", authority / "ca.key" + csr, key, cert = authority / "client.csr", client / "tls.key", client / "tls.crt" + try: + crypto("req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", + "-nodes", "-keyout", ca_key, "-out", ca, "-days", "1", + "-subj", "/O=openshell/OU=SAW-apply/CN=saw-apply-ca", + "-addext", "basicConstraints=critical,CA:TRUE", "-addext", "keyUsage=critical,keyCertSign") + crypto("req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", + "-nodes", "-keyout", key, "-subj", subject, "-out", csr) + ext = authority / "ext.cnf" + ext.write_text("basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\n" + "extendedKeyUsage=clientAuth\n", encoding="utf-8") + crypto("x509", "-req", "-in", csr, "-CA", ca, "-CAkey", ca_key, + "-set_serial", "0x"+secrets.token_hex(16), "-days", "1", "-extfile", ext, "-out", cert) + finally: + if ca_key.exists(): + erase(ca_key) + return ca.read_text(encoding="utf-8") + + +@contextmanager +def identity(server_ca, runtime, *, authority_base=None, subject="/O=openshell/OU=openshell-admin/CN=saw-installer", + owner=None, require_tmpfs=True, crypto=openssl, erase=erase_ca_key): + """Create the same private directory layout and cleanup contract as the VM.""" + runtime = Path(runtime) + if not runtime.is_dir(): + raise AdminSessionError("runtime tmpfs is unavailable") + if require_tmpfs: + require_memory(runtime) + authority_base = Path(authority_base) if authority_base else runtime / "saw-authorities" + session_dir = root_dir = None + try: + session_dir = Path(tempfile.mkdtemp(prefix="saw-admin.", dir=runtime)) + authority_base.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(authority_base, 0o700) + root_dir = Path(tempfile.mkdtemp(prefix="session.", dir=authority_base)) + tls, config = session_dir / "tls", session_dir / "config" + client = tls / "client" + client.mkdir(parents=True) + config.mkdir() + for path in (session_dir, tls, client, config): + os.chmod(path, 0o700) + if owner and os.geteuid() == 0: + os.chown(path, *owner) + ca_pem = issue(root_dir, client, subject=subject, crypto=crypto, erase=erase) + (tls / "ca.crt").write_bytes(server_ca) + for path in (client / "tls.crt", client / "tls.key", tls / "ca.crt"): + os.chmod(path, 0o600) + if owner and os.geteuid() == 0: + os.chown(path, *owner) + yield {"id": str(uuid.uuid4()), "ca": ca_pem, "tls": tls, "config": config, + "expires": int(time.time())+86400} + finally: + for path in (session_dir, root_dir): + if path is not None: + shutil.rmtree(path, ignore_errors=True) + + +def wait_for_trust(probe, *, timeout=180, sleep=time.sleep, monotonic=time.monotonic): + """Observe authenticated acceptance, rather than assuming an API write reloaded TLS.""" + deadline = monotonic()+timeout + while True: + if probe(): + return + if monotonic() >= deadline: + raise AdminSessionError("gateway did not accept the per-run admin CA before the deadline") + sleep(1) diff --git a/charts/openshell-saw/files/installer/saw_core.py b/charts/openshell-saw/files/installer/saw_core.py new file mode 100644 index 000000000..c74428747 --- /dev/null +++ b/charts/openshell-saw/files/installer/saw_core.py @@ -0,0 +1,1565 @@ +"""Shared SAW profile models, validation and workspace dependency ordering. + +This is the canonical source for VM and Kubernetes integrations. It owns no +subprocess, cluster, credential-resolution or installation operations; effectful +ordering callbacks belong to backend adapters. Runtime +credential fields on legacy dataclasses remain for compatibility; parsing only +populates declarative references, never credential material. Backend capability +checks and mutation adapters are separate from this common profile language. + +The VM chart embeds these exact bytes into its signed installer at render time. +Do not maintain a second copy or import this module from untrusted input paths. +""" + +import base64 +import hashlib +import json +import re +import shlex +from urllib.parse import urlsplit +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +API_VERSION = "saw.redhat.com/v1alpha1" + + +NAME_RE = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") + + +SECRET_NAME_RE = re.compile(r"^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$") + + +SECRET_KEY_RE = re.compile(r"^[-._a-zA-Z0-9]+$") + + +OPENSHELL_NAME_LIMIT = 19 + + +PROVIDER_CRED_MAP = { + "gemini": "GEMINI_API_KEY", + "google-vertex-ai": "GOOGLE_API_KEY", + "claude-code": "ANTHROPIC_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "codex": "OPENAI_API_KEY", + "openai": "OPENAI_API_KEY", + "nvidia": "NVIDIA_API_KEY", + "build": "NVIDIA_INFERENCE_API_KEY", + "brave": "BRAVE_API_KEY", + "tavily": "TAVILY_API_KEY", +} + + +# NemoClaw calls the NVIDIA API Catalog "build"; OpenShell calls it "nvidia". +NEMOCLAW_PROVIDER_MAP = { + "nvidia": "build", +} + + +def nemoclaw_provider(provider): + return provider.nemoclaw_provider or NEMOCLAW_PROVIDER_MAP.get(provider.type, provider.type) + + +def profile_credential_envs(profile_text): + """The first env var of each credential a provider profile declares, in + order (credentials[i].env_vars[0]); "" for a credential without one.""" + try: + doc = yaml.safe_load(profile_text) if profile_text else None + except yaml.YAMLError: + return [] + out = [] + for cred in (doc or {}).get("credentials") or []: + envs = (cred.get("env_vars") or []) if isinstance(cred, dict) else [] + out.append(envs[0] if envs else "") + return out + + +def profile_credentials(profile_text): + """A provider profile's credentials: [(name, [env vars])], in order.""" + try: + doc = yaml.safe_load(profile_text) if profile_text else None + except yaml.YAMLError: + return [] + out = [] + for cred in (doc or {}).get("credentials") or []: + if isinstance(cred, dict) and cred.get("name"): + out.append((str(cred["name"]), [str(e) for e in cred.get("env_vars") or []])) + return out + + +def profile_all_credential_envs(profile_text): + """Every env var any credential of a provider profile is exposed under + (all of credentials[].env_vars, e.g. GITHUB_TOKEN and GH_TOKEN).""" + try: + doc = yaml.safe_load(profile_text) if profile_text else None + except yaml.YAMLError: + return set() + return {env for cred in (doc or {}).get("credentials") or [] if isinstance(cred, dict) + for env in (cred.get("env_vars") or [])} + + +def profile_credential_env(profile_text): + """The primary credential env var a provider profile declares + (credentials[0].env_vars[0]), or "" when it declares none.""" + envs = profile_credential_envs(profile_text) + return envs[0] if envs else "" + + +def credential_marker(prefix, secret_name, key): + """Opaque value that the trusted proxy replaces for an approved endpoint.""" + return f"{prefix}.{secret_name}.{key}" + + +def credential_env_name(provider_type, profiles): + """Env var the installer passes a provider's credential through: the one + the provider's own profile declares, else the built-in default. This lets + a profile-backed type (github, slack, gmail, ...) work as a provider with + no PROVIDER_CRED_MAP edit; for the shipped built-ins the two agree.""" + return (profile_credential_env((profiles or {}).get(provider_type, "")) + or PROVIDER_CRED_MAP.get(provider_type, "")) + + +SANDBOX_TYPES = {"generic", "openclaw", "nemoclaw"} + + +BASE_URL_CONFIG_KEYS = { + "openai": "OPENAI_BASE_URL", + "anthropic": "ANTHROPIC_BASE_URL", + "nvidia": "NVIDIA_BASE_URL", +} + + +# OpenShell 0.1.x has no inference router: the agent calls the endpoint +# itself, and the provider profile must name that endpoint's host. +# OpenShell 0.1.x removed managed inference (`openshell inference`, +# https://inference.local). An agent is configured with its provider's native +# OpenAI-compatible endpoint and the placeholder key the sandbox receives in +# the provider's env var; the sandbox proxy swaps in the real key only for +# requests to the profile's endpoints from the profile's binaries. A +# provider's baseUrl overrides these. +NATIVE_BASE_URLS = { + "nvidia": "https://integrate.api.nvidia.com/v1", + "build": "https://integrate.api.nvidia.com/v1", + "openai": "https://api.openai.com/v1", +} + +OPENCLAW_AUTH_CHOICES = {"openai": "openai-api-key", "anthropic": "apiKey"} +# By protocol: the official API an agent talks to when no endpoint is given. +PROTOCOL_NATIVE_BASE_URLS = {"openai": "https://api.openai.com/v1", + "anthropic": "https://api.anthropic.com"} + + +class InstallerError(Exception): + """A failure with a message that is safe to log and show in status.""" + + +@dataclass +class Provider: + name: str + type: str + enabled: bool = True + nemoclaw_provider: str = "" + credential_secret: str = "" + credential_secret_key: str = "api_key" + model: str = "" + # Secret keys the base URL and the model are read from (optional). + base_url_secret_key: str = "" + model_secret_key: str = "" + inference_timeout: int = 0 + base_url: str = "" + credential_free: bool = False + # Optional `credentials:` mapping, keyed by the profile's credential name + # ({api_key: {secretKey: ...}, app_secret: {secretKey: ...}}): one value + # per credential, from the same Secret, each passed as its own + # `--credential ENV` with the env var that credential declares. The + # single-credential shorthand credentialSecretKey is one entry for the + # profile's first credential. + refresh_strategy: str = "" + refresh_material: dict = field(default_factory=dict) + credentials_spec: dict = field(default_factory=dict) + credential_values: dict = field(default_factory=dict) # credential name -> value + + +@dataclass +class Sandbox: + name: str + type: str = "generic" + enabled: bool = True + agent: str = "openclaw" + image: str = "" + providers: list = field(default_factory=list) + model: str = "" + harness_ref: dict = field(default_factory=dict) + ui: dict = field(default_factory=dict) + # `env:` -- the workload's environment (`sandbox create --env`; changing + # it recreates the sandbox). {NAME: {value, title?}}: literals only, so + # env_values (below) is derived, never a resolved copy that could go + # stale (found in CI: freshly parsed profiles compared an empty copy + # with the sandbox's env label and recreated it). Never for credentials + # (the agent, and argv, can read these): those stay on providers. + env: dict = field(default_factory=dict) + # `inference:` -- how the agent reaches its model: {providerRef, protocol, + # baseUrl?, model, models?, timeoutSeconds?}. models maps optional native + # provider names to additional chat model inputs. baseUrl/model/timeoutSeconds + # is a literal or an input ({secretKeyRef: {name, key}, default?, + # required?, title?}); inference_values is it resolved. The installer onboards the + # agent from it (by protocol, so any profile-backed provider type works). + inference: dict = field(default_factory=dict) + inference_values: dict = field(default_factory=dict) + # Container-level GPU request (`openshell sandbox create --gpu`). + # Independent of the gateway VM's vm.gpu.enabled (KubeVirt hostDevices). + # Off unless the BOM sandbox sets gpu.enabled. See docs/gpu-passthrough.md. + gpu_enabled: bool = False + gpu_count: int = 1 + + @property + def env_values(self): + """NAME -> value for the sandbox's env (literal entries; anything + else is refused by validation).""" + if not isinstance(self.env, dict): + return {} + return {str(name): str(src["value"]) for name, src in sorted(self.env.items()) + if isinstance(src, dict) and "value" in src} + + +@dataclass +class Workspace: + name: str + enabled: bool = True + description: str = "" + providers: list = field(default_factory=list) + sandboxes: list = field(default_factory=list) + + +@dataclass +class Profile: + name: str + workspaces: list = field(default_factory=list) + + +@dataclass +class HarnessBundle: + name: str + agent: str = "openclaw" + digest: str = "" + files: dict = field(default_factory=dict) # relpath -> base64 text + + +def read_profile_files(directory): + """Read the flattened saw-bom ConfigMap. + + Returns (profile files as text, harness files as bytes, harness index). + Profile keys are profiles______.yaml; harness keys are + harness____ and hold base64 (see the digest + contract in templates/configmap-bom.yaml and parse_harness_files). + """ + directory = Path(directory) + if not directory.is_dir(): + return {}, {}, {} + return split_bom_files({p.name: p.read_text(encoding="utf-8") for p in sorted(directory.iterdir()) + if not p.name.startswith(".") and p.is_file()}) + + +def split_bom_files(data): + """Decode one flat saw-bom ConfigMap from either VM disk or Kubernetes API.""" + files, harness, index = {}, {}, {} + for key, value in data.items(): + if key.startswith("profiles__") and key.endswith(".yaml"): + files[key] = value + elif key.startswith("harness__"): + try: + harness[key] = base64.b64decode( + value, validate=True) + except (ValueError, UnicodeDecodeError) as exc: + raise InstallerError( + f"harness file {key} is not valid base64 ({exc}); " + "the chart must b64enc every harness key") from None + elif key == "harness-index.yaml": + index = _yaml(value, key) + else: + # A wrong key layout must not look like "no profiles". + raise InstallerError(f"unexpected file in the profiles ConfigMap: {key}") + return files, harness, index + + +def _yaml(text, where): + try: + return yaml.safe_load(text) or {} + except yaml.YAMLError as exc: + raise InstallerError(f"{where}: invalid YAML ({exc})") from None + + +PROFILE_DOCUMENT_KINDS = { + "workspace.yaml": "Workspace", "providers.yaml": "Providers", "sandbox.yaml": "Sandboxes", +} +PROFILE_PROVIDER_FIELDS = { + "name", "type", "enabled", "credentialFree", "nemoclawProvider", "credentialSecret", + "credentialSecretKey", "model", "baseUrlSecretKey", "modelSecretKey", "inferenceTimeout", + "refresh", "credentials", +} +PROFILE_SANDBOX_FIELDS = { + "name", "type", "enabled", "agent", "image", "providers", "model", "harnessRef", "env", "inference", + "harnessRequired", "ui", "gpu", +} + + +def _sandbox_gpu(s, key): + gpu = s.get("gpu", {}) + if not isinstance(gpu, dict): + raise InstallerError(f"{key}: sandbox '{s['name']}' gpu must be a mapping") + unknown = sorted(str(key) for key in gpu if key not in {"enabled", "count"}) + if unknown: + raise InstallerError(f"{key}: sandbox '{s['name']}' gpu has unknown keys {', '.join(unknown)}") + if not isinstance(gpu.get("enabled", False), bool): + raise InstallerError(f"{key}: sandbox '{s['name']}' gpu.enabled must be true or false") + gpu_count = gpu.get("count", 1) + if isinstance(gpu_count, bool) or not isinstance(gpu_count, int): + raise InstallerError(f"{key}: sandbox '{s['name']}' gpu.count must be an integer") + return gpu.get("enabled", False), gpu_count + + +def _profile_mapping(value, allowed, required, where): + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise InstallerError(f"{where}: expected a mapping with string keys") + missing, unknown = set(required) - set(value), set(value) - set(allowed) + if missing or unknown: + raise InstallerError(f"{where}: unknown or missing profile fields") + return value + + +def _profile_input(value, kinds, where): + if isinstance(value, dict): + _profile_mapping(value, INPUT_KEYS, {"secretKeyRef"}, where) + reference = _profile_mapping(value["secretKeyRef"], {"name", "key"}, {"name", "key"}, where) + if any(not isinstance(item, str) for item in reference.values()): + raise InstallerError(f"{where}: input references must be strings") + if "title" in value and not isinstance(value["title"], str): + raise InstallerError(f"{where}: input title must be a string") + if input_problems(value, kinds): + raise InstallerError(f"{where}: invalid inference input") + + +def validate_profile_document(doc, kind, where="profile"): + """Strict common document schema, independent of backend capabilities. + + Legacy VM parsing remains compatible by default; new adapters call + ``parse_profiles(..., strict=True)`` before capability checks so unknown + fields cannot be silently discarded while constructing dataclasses. + ``validate_profiles`` still validates cross-object semantics afterward. + """ + _profile_mapping(doc, {"apiVersion", "kind", "metadata", "spec"}, {"apiVersion", "kind"}, where) + if kind not in PROFILE_DOCUMENT_KINDS.values() or doc["apiVersion"] != API_VERSION or doc["kind"] != kind: + raise InstallerError(f"{where}: unsupported profile apiVersion or kind") + metadata = _profile_mapping(doc.get("metadata", {}), {"name", "description", "profile"}, set(), where) + if any(not isinstance(value, str) for value in metadata.values()): + raise InstallerError(f"{where}: metadata fields must be strings") + fields = {"Workspace": {"enabled"}, "Providers": {"providers"}, "Sandboxes": {"sandboxes"}}[kind] + spec = _profile_mapping(doc.get("spec", {}), fields, set(), where) + if kind == "Workspace": + if "enabled" in spec and type(spec["enabled"]) is not bool: + raise InstallerError(f"{where}: enabled must be boolean") + return doc + list_key = "providers" if kind == "Providers" else "sandboxes" + entries = spec.get(list_key, []) + if not isinstance(entries, list): + raise InstallerError(f"{where}: {list_key} must be a list") + allowed = PROFILE_PROVIDER_FIELDS if kind == "Providers" else PROFILE_SANDBOX_FIELDS + required = {"name", "type"} if kind == "Providers" else {"name"} + for entry in entries: + _profile_mapping(entry, allowed, required, where) + booleans = {"enabled", "credentialFree"} if kind == "Providers" else {"enabled", "harnessRequired"} + objects = {"refresh", "credentials"} if kind == "Providers" else {"harnessRef", "env", "inference", "ui"} + for field_name, value in entry.items(): + if field_name in booleans: + if type(value) is not bool: + raise InstallerError(f"{where}: {field_name} must be boolean") + elif field_name in objects: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise InstallerError(f"{where}: {field_name} must be a mapping with string keys") + elif field_name == "gpu": + _sandbox_gpu(entry, where) + elif field_name == "inferenceTimeout": + if type(value) is not int or value < 0: + raise InstallerError(f"{where}: inferenceTimeout must be a nonnegative integer") + elif field_name == "providers": + if not isinstance(value, list) or any(not isinstance(ref, str) for ref in value): + raise InstallerError(f"{where}: providers must be a list of names") + elif not isinstance(value, str): + raise InstallerError(f"{where}: {field_name} must be a string") + if kind == "Providers": + if "refresh" in entry: + refresh = _profile_mapping(entry["refresh"], {"strategy"}, {"strategy"}, where) + if not isinstance(refresh["strategy"], str): + raise InstallerError(f"{where}: refresh.strategy must be a string") + if "credentials" in entry: + for credential in entry["credentials"].values(): + _profile_mapping(credential, {"secretKey", "title"}, {"secretKey"}, where) + if any(not isinstance(value, str) for value in credential.values()): + raise InstallerError(f"{where}: credential references must be strings") + else: + if "ui" in entry: + ui = _profile_mapping(entry["ui"], {"route"}, set(), where) + if "route" in ui and type(ui["route"]) is not bool: + raise InstallerError(f"{where}: ui.route must be boolean") + if "harnessRef" in entry: + harness = _profile_mapping(entry["harnessRef"], {"name", "image", "digest"}, set(), where) + if any(not isinstance(value, str) for value in harness.values()): + raise InstallerError(f"{where}: harness references must be strings") + for name, value in entry.get("env", {}).items(): + _profile_mapping(value, SANDBOX_ENV_KEYS, {"value"}, where) + if "title" in value and not isinstance(value["title"], str): + raise InstallerError(f"{where}: environment title must be a string") + if sandbox_env_problems(name, value): + raise InstallerError(f"{where}: invalid sandbox environment entry") + inference = entry.get("inference", {}) + _profile_mapping(inference, INFERENCE_KEYS, set(), where) + if "providerRef" in inference and not isinstance(inference["providerRef"], str): + raise InstallerError(f"{where}: inference.providerRef must be a string") + if "protocol" in inference and not isinstance(inference["protocol"], str): + raise InstallerError(f"{where}: inference.protocol must be a string") + for field_name in ("model", "baseUrl", "timeoutSeconds"): + kinds = (str, int) if field_name == "timeoutSeconds" else (str,) + if field_name in inference: + _profile_input(inference[field_name], kinds, where) + if "models" in inference: + models = inference["models"] + if not isinstance(models, dict) or any(not isinstance(key, str) for key in models): + raise InstallerError(f"{where}: inference.models must be a mapping of names to inputs") + for value in models.values(): + _profile_input(value, (str,), where) + return doc + + +def parse_profiles(files, *, strict=False, document_check=None): + """Build profiles from {flat key: yaml text}. + + Only keys of the form profiles______.yaml are + used, where is workspace, providers or sandbox. Strict mode validates + the common document shape before normalization; backend capabilities remain + a separate check, so all backends use the same profile language. An optional + ``document_check(doc, kind)`` can reject backend-unsupported chart directives + before normalization (for example UI routes managed outside the VM parser). + It receives a schema-validated document whenever strict mode is enabled. + """ + grouped = {} + for key, text in files.items(): + parts = key.split("__") + if len(parts) != 4 or parts[0] != "profiles": + raise InstallerError(f"unexpected profile file name: {key}") + _, profile, ws_dir, filename = parts + if filename not in ("workspace.yaml", "providers.yaml", "sandbox.yaml"): + raise InstallerError(f"unexpected profile file: {key}") + if strict or document_check is not None: + doc, kind = _yaml(text, key), PROFILE_DOCUMENT_KINDS[filename] + if strict: + validate_profile_document(doc, kind, key) + if document_check is not None: + document_check(doc, kind) + grouped.setdefault(profile, {}).setdefault(ws_dir, {})[filename] = (key, text) + + profiles = [] + for profile_name in sorted(grouped): + profile = Profile(name=profile_name) + for ws_dir in sorted(grouped[profile_name]): + docs = grouped[profile_name][ws_dir] + if "workspace.yaml" not in docs: + raise InstallerError(f"profile {profile_name}/{ws_dir} has no workspace.yaml") + key, text = docs["workspace.yaml"] + ws_doc = _yaml(text, key) + meta = ws_doc.get("metadata") or {} + spec = ws_doc.get("spec") or {} + ws = Workspace(name=meta.get("name", ws_dir), + enabled=spec.get("enabled", True), + description=meta.get("description", "")) + if "providers.yaml" in docs: + key, text = docs["providers.yaml"] + for p in (_yaml(text, key).get("spec") or {}).get("providers") or []: + if "name" not in p or "type" not in p: + raise InstallerError(f"{key}: every provider needs name and type") + cred_spec = p.get("credentials") + if cred_spec is not None and "credentialSecretKey" in p: + raise InstallerError(f"{key}: provider '{p['name']}' sets both " + "credentials and credentialSecretKey; use one") + primary_key = p.get("credentialSecretKey", "api_key") + ws.providers.append(Provider( + name=p["name"], type=p["type"], + enabled=p.get("enabled", True), + credential_free=p.get("credentialFree", False), + nemoclaw_provider=p.get("nemoclawProvider", ""), + credential_secret=p.get("credentialSecret", ""), + credential_secret_key=primary_key, + model=p.get("model", ""), + base_url_secret_key=p.get("baseUrlSecretKey", ""), + model_secret_key=p.get("modelSecretKey", ""), + inference_timeout=int(p.get("inferenceTimeout", 0) or 0), + refresh_strategy=str((p.get("refresh") or {}).get("strategy", "")), + credentials_spec=cred_spec if cred_spec is not None else {})) + if "sandbox.yaml" in docs: + key, text = docs["sandbox.yaml"] + for s in (_yaml(text, key).get("spec") or {}).get("sandboxes") or []: + if "name" not in s: + raise InstallerError(f"{key}: every sandbox needs a name") + gpu_enabled, gpu_count = _sandbox_gpu(s, key) + ws.sandboxes.append(Sandbox( + name=s["name"], type=s.get("type", "generic"), + enabled=s.get("enabled", True), + agent=s.get("agent", "openclaw"), + image=s.get("image", ""), + providers=list(s.get("providers") or []), + model=s.get("model", ""), + harness_ref=dict(s.get("harnessRef") or {}), + ui=dict(s.get("ui") or {}), + env=s.get("env") if s.get("env") is not None else {}, + inference=s.get("inference") if s.get("inference") is not None else {}, + gpu_enabled=gpu_enabled, gpu_count=gpu_count)) + profile.workspaces.append(ws) + profiles.append(profile) + return profiles + + +def enabled_workspaces(profiles): + for profile in profiles: + for ws in profile.workspaces: + if ws.enabled: + yield profile, ws + + +def apply_workspace_sequence(workspace, providers, sandboxes, adapter): + """Shared dependency ordering for one bounded workspace apply pass. + + The backend supplies ``prepare_workspace``, ``apply_provider`` and + ``apply_sandbox`` methods returning readiness booleans. Observe/apply all + providers before the barrier; a pending workspace or provider prevents + sandbox creation. Backends retain authentication, ownership, capability + validation, retry/deletion policy and error handling. No backend exception + is swallowed here, and this function does not turn readiness into agent + health. Callers select enabled entries or implement explicit skip behavior. + """ + if not adapter.prepare_workspace(workspace): + return False + providers_ready = True + for provider in providers: + if not adapter.apply_provider(workspace, provider): + providers_ready = False + if not providers_ready: + return False + sandboxes_ready = True + for sandbox in sandboxes: + if not adapter.apply_sandbox(workspace, sandbox): + sandboxes_ready = False + return sandboxes_ready + + +def validate_profiles(profiles, known_profiles=(), governed=False): + """Catch mistakes before anything touches the gateway. + + governed: the gateway takes its provider profiles from the governance + interceptor, which may serve profiles the installer does not ship + (overrides/governance-profiles). A type that is neither built-in nor + shipped is then checked against the live catalog at apply time + (ProfileApplier.adopt_governed_profiles) instead of refused here.""" + errors = [] + seen = {} + for profile, ws in enabled_workspaces(profiles): + where = f"{profile.name}/{ws.name}" + if not NAME_RE.match(ws.name) or len(ws.name) > OPENSHELL_NAME_LIMIT: + errors.append(f"{where}: workspace name must be a DNS label of at most {OPENSHELL_NAME_LIMIT} characters") + if ws.name in seen: + errors.append(f"{where}: workspace '{ws.name}' is also defined by profile {seen[ws.name]}") + seen[ws.name] = profile.name + provider_names = [p.name for p in ws.providers if p.enabled] + provider_types = {p.name: p.type for p in ws.providers if p.enabled} + if len(provider_names) != len(set(provider_names)): + errors.append(f"{where}: duplicate provider names") + for p in ws.providers: + if not p.enabled: + continue + if not isinstance(p.credential_free, bool): + errors.append(f"{where}: provider '{p.name}' credentialFree must be a boolean") + elif p.credential_free: + # A keyless provider is an attachment to a governed service + # (an in-cluster MCP Service): no key, no model, and a type + # that is a live governance profile, not a built-in type. + for label, value in (("name", p.name), ("type", p.type)): + if not NAME_RE.match(value or "") or len(value) > OPENSHELL_NAME_LIMIT: + errors.append(f"{where}: keyless provider {label} must be a " + f"DNS label of at most {OPENSHELL_NAME_LIMIT} characters") + if p.type in PROVIDER_CRED_MAP: + errors.append(f"{where}: provider '{p.name}' credentialFree cannot disable " + f"the credential requirement of built-in type '{p.type}'") + if p.credential_secret: + errors.append(f"{where}: provider '{p.name}' credentialFree cannot be " + "combined with credentialSecret") + if (p.model or p.nemoclaw_provider or p.model_secret_key or p.base_url_secret_key + or p.base_url or p.inference_timeout): + errors.append(f"{where}: keyless provider '{p.name}' cannot be combined " + "with model/inference settings") + if governed and p.name != p.type: + # OpenShell 0.1.2's governance interceptor validates a new + # sandbox's providers by comparing each provider *name* with + # the vended profile ids (examples/governance-interceptor, + # providers_are_managed): a provider named differently from + # its type is refused at sandbox create with "sandbox + # providers may only use vended provider profiles". Found + # live; refuse it here, with the reason, instead. + errors.append(f"{where}: provider '{p.name}' must be named after its type " + f"'{p.type}' (the governance interceptor checks sandbox providers " + "by profile id, so name and type must match)") + if (not p.credential_free and p.type not in PROVIDER_CRED_MAP + and p.type not in known_profiles): + if not governed: + errors.append(f"{where}: provider '{p.name}' has unsupported type '{p.type}' " + "(no built-in type and no shipped provider profile)") + elif not NAME_RE.match(p.type or "") or len(p.type) > OPENSHELL_NAME_LIMIT: + errors.append(f"{where}: provider '{p.name}' type must be a DNS label of at " + f"most {OPENSHELL_NAME_LIMIT} characters (a governance profile id)") + if not p.credential_secret and not p.credential_free: + errors.append(f"{where}: provider '{p.name}' has no credentialSecret") + elif p.credential_secret and not SECRET_NAME_RE.match(p.credential_secret): + errors.append(f"{where}: provider '{p.name}' has an invalid credentialSecret name") + if not SECRET_KEY_RE.match(p.credential_secret_key or ""): + errors.append(f"{where}: provider '{p.name}' has an invalid credentialSecretKey") + for field_name, value in (("baseUrlSecretKey", p.base_url_secret_key), + ("modelSecretKey", p.model_secret_key)): + if value and not SECRET_KEY_RE.match(value): + errors.append(f"{where}: provider '{p.name}' has an invalid {field_name}") + if p.base_url_secret_key and p.type not in BASE_URL_CONFIG_KEYS: + errors.append(f"{where}: provider '{p.name}' of type '{p.type}' does not take a base URL " + f"(supported: {', '.join(sorted(BASE_URL_CONFIG_KEYS))})") + if p.inference_timeout < 0: + errors.append(f"{where}: provider '{p.name}' has a negative inferenceTimeout") + if p.refresh_strategy and p.refresh_strategy != "oauth2_refresh_token": + errors.append(f"{where}: provider '{p.name}' has unsupported refresh strategy '{p.refresh_strategy}'") + if p.credentials_spec: + if not isinstance(p.credentials_spec, dict): + errors.append(f"{where}: provider '{p.name}' credentials must be a mapping of " + "the profile's credential name to {secretKey[, title]}") + elif p.credential_free: + errors.append(f"{where}: keyless provider '{p.name}' cannot have credentials") + else: + for cname, entry in p.credentials_spec.items(): + if not CREDENTIAL_NAME_RE.match(str(cname)): + errors.append(f"{where}: provider '{p.name}' credential '{cname}' is " + "not a valid credential name") + if not isinstance(entry, dict) or not SECRET_KEY_RE.match( + str(entry.get("secretKey", ""))): + errors.append(f"{where}: provider '{p.name}' credential '{cname}' " + "needs a valid secretKey") + elif set(entry) - {"secretKey", "title"}: + errors.append(f"{where}: provider '{p.name}' credential '{cname}' has " + "unknown keys " + f"{', '.join(sorted(set(entry) - {'secretKey', 'title'}))}") + sandbox_names = set() + for s in ws.sandboxes: + if not s.enabled: + continue + if s.name in sandbox_names: + errors.append(f"{where}: duplicate sandbox '{s.name}'") + sandbox_names.add(s.name) + if len(s.name) > OPENSHELL_NAME_LIMIT or not NAME_RE.match(s.name): + errors.append(f"{where}: sandbox name '{s.name}' must be a DNS label of at most {OPENSHELL_NAME_LIMIT} characters") + if s.gpu_enabled and s.gpu_count < 1: + errors.append(f"{where}: sandbox '{s.name}' gpu.count must be at least 1") + if s.type not in SANDBOX_TYPES: + errors.append(f"{where}: sandbox '{s.name}' has unsupported type '{s.type}'") + for ref in s.providers: + if ref not in provider_names: + errors.append(f"{where}: sandbox '{s.name}' uses provider '{ref}' which is not an enabled provider in this workspace") + if s.type in ("openclaw", "nemoclaw") and not provider_names: + errors.append(f"{where}: {s.type} sandbox '{s.name}' needs at least one provider") + if s.inference: + errors += [f"{where}: sandbox '{s.name}' {p}" + for p in sandbox_inference_problems( + s, provider_types, {p.name: p.type for p in ws.providers})] + if not isinstance(s.env, dict): + errors.append(f"{where}: sandbox '{s.name}' env must be a mapping of NAME to " + "{value} or {secret, secretKey}") + else: + for name, src in s.env.items(): + errors += [f"{where}: sandbox '{s.name}' env '{name}' {problem}" + for problem in sandbox_env_problems(name, src)] + if errors: + raise InstallerError("invalid profiles:\n - " + "\n - ".join(errors)) + + +# Match OpenShell's environment-name and per-value size constraints. +ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +CREDENTIAL_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") + + +ENV_VALUE_LIMIT = 8192 + + +SANDBOX_ENV_KEYS = {"value", "title"} + + +INPUT_KEYS = {"secretKeyRef", "default", "required", "title"} + + +def secret_ref(src): + """(name, key) of a {secretKeyRef: {name, key}} input, or None.""" + ref = src.get("secretKeyRef") if isinstance(src, dict) else None + if not isinstance(ref, dict): + return None + return str(ref.get("name", "")), str(ref.get("key", "")) + + +def secret_ref_problems(src): + ref = secret_ref(src) + if ref is None: + return ["needs secretKeyRef: {name, key}"] + out = [] + if not SECRET_NAME_RE.match(ref[0]): + out.append("secretKeyRef.name must be a valid Secret name") + if not SECRET_KEY_RE.match(ref[1]): + out.append("secretKeyRef.key must be a valid Secret key") + if set(src["secretKeyRef"]) - {"name", "key"}: + out.append("secretKeyRef takes only name and key") + return out + + +INFERENCE_KEYS = {"providerRef", "protocol", "baseUrl", "model", "models", "timeoutSeconds"} + + +# A profile controls allowed credential destinations; the protocol controls +# agent onboarding independently of a governed provider's custom profile name. +INFERENCE_PROTOCOLS = {"openai", "anthropic"} + + +PROVIDER_PROTOCOLS = {"openai": "openai", "codex": "openai", "nvidia": "openai", + "build": "openai", "anthropic": "anthropic", "claude-code": "anthropic"} + + +def check_env_value(value): + """A value OpenShell accepts for `--env`: one line, at most 8 KiB. The + value is not echoed (it may come from a Secret).""" + if "\n" in value or "\r" in value or "\0" in value: + raise ValueError("value must be a single line") + if len(value.encode()) > ENV_VALUE_LIMIT: + raise ValueError(f"value is longer than {ENV_VALUE_LIMIT} bytes") + return value + + +def sandbox_env_problems(name, src): + """What is wrong with one `env:` entry of a sandbox ([] when fine).""" + out = [] + if not ENV_NAME_RE.match(str(name)): + out.append("is not a valid variable name ([A-Za-z_][A-Za-z0-9_]*)") + elif str(name).startswith("OPENSHELL_"): + out.append("is reserved by OpenShell (OPENSHELL_*)") + elif name in set(PROVIDER_CRED_MAP.values()): + out.append("is a provider credential variable; credentials go on a provider, " + "the agent can read sandbox env") + elif name in set(BASE_URL_CONFIG_KEYS.values()) or name in ("MODEL", "OPENCLAW_MODEL", + "NEMOCLAW_MODEL"): + out.append("is an inference setting; set it in the sandbox's inference: block") + if isinstance(src, dict) and ("secretKeyRef" in src or "default" in src): + # A value from a Secret would travel in `sandbox create --env`'s argv + # and be readable by every process in the sandbox (review, P1): + # secrets go on a provider, whose profile the gateway injects. + return out + ["must be a literal ({value: ...}); a value from a Secret belongs on a " + "provider, not in sandbox env (the agent and argv could read it)"] + if not isinstance(src, dict) or "value" not in src: + return out + ["needs {value: ...}"] + unknown = sorted(set(src) - SANDBOX_ENV_KEYS) + if unknown: + out.append(f"has unknown keys {', '.join(unknown)}") + if isinstance(src["value"], bool) or not isinstance(src["value"], (str, int, float)): + out.append("value must be a string (quote it)") + else: + try: + check_env_value(str(src["value"])) + except ValueError as exc: + out.append(f"value: {exc}") + return out + + +def input_problems(src, kinds=(str,)): + """What is wrong with one typed input (a literal or {secret, secretKey, + default?, required?, title?}); [] when fine.""" + if not isinstance(src, dict): + return [] if isinstance(src, kinds) and not isinstance(src, bool) else [ + "must be a string or {secretKeyRef: {name, key}, ...}"] + out = [] + unknown = sorted(set(src) - INPUT_KEYS) + if unknown: + out.append(f"has unknown keys {', '.join(unknown)}") + out += secret_ref_problems(src) + if "required" in src and not isinstance(src["required"], bool): + out.append("required must be true or false") + if "default" in src and (isinstance(src["default"], bool) + or not isinstance(src["default"], kinds)): + out.append("default must be a string") + return out + + +def inference_protocol(spec, provider_type): + """The protocol an `inference:` block uses: its own `protocol`, else the + one the provider's type implies ("" when neither says).""" + return spec.get("protocol") or PROVIDER_PROTOCOLS.get(provider_type or "", "") + + +SHELL_UNSAFE = set("\"'`$\\;|&<>(){}") + +def check_base_url(url): + """An http(s) base URL without credentials, query or fragment. The value + is not echoed in errors: a pasted URL may contain a token.""" + try: + parts = urlsplit(url) + ok = (parts.scheme in ("http", "https") and parts.hostname + and parts.username is None and parts.password is None + and not parts.query and not parts.fragment + and not any(c.isspace() or c in SHELL_UNSAFE for c in url)) + if ok: + parts.port # raises ValueError on a bad port + except ValueError: + ok = False + if not ok: + raise ValueError("base URL must be http(s)://host[:port][/path] without credentials, " + "query or fragment") + host = parts.hostname + if host in ("localhost", "127.0.0.1", "::1"): + raise ValueError("base URL must not be localhost: inside the sandbox that is the " + "sandbox itself (use a cluster Service or Route host)") + return url.rstrip("/") + + +def resolve_bom_input(src, read_value, what, forbidden=()): + """A typed input's value: the literal, or the Secret key it refers to + (secretKeyRef), else its default; "" when optional and absent.""" + if not isinstance(src, dict): + return str(src) + sname, skey = secret_ref(src) + if (sname, skey) in forbidden: + raise InstallerError(f"{what}: secretKeyRef names a provider credential or refresh material; refusing to expose it") + value = read_value(sname, skey) + if value: + return value + if "default" in src: + return str(src["default"]) + if src.get("required", "default" not in src): + raise InstallerError(f"{what} not found: Secret '{sname}' key '{skey}'. List the " + "Secret in openshell-saw additionalProviderSecrets and make sure " + "it exists.") + return "" + + +def select_inference_provider(providers, names, *, inference_ref=None): + """Select an agent provider from the adapter's available candidates. + + Disabled and credential-free providers cannot onboard an agent. An explicit + inference reference never falls back to a different provider. Otherwise use + attachment order, preferring a candidate with a model. The legacy empty + attachment list retains candidate order; strict adapters must validate + attachment requirements before calling. Availability and governance remain + adapter checks, so a skipped provider must be absent from ``providers``. + """ + usable = [p for p in providers if p.enabled and not p.credential_free] + if inference_ref is not None: + return next((p for p in usable if p.name == inference_ref), None) + if names: + usable = [p for name in names for p in usable if p.name == name] + return next((p for p in usable if p.model), usable[0] if usable else None) + + +def sandbox_protocol(sb, provider): + """How the agent talks to its inference endpoint.""" + if sb.inference_values: + return sb.inference_values["protocol"] + return PROVIDER_PROTOCOLS.get(provider.type, "") if provider else "" + + +def sandbox_model(sb, provider, default=""): + """The model the agent runs: `inference.model`, else the legacy sandbox + `model`, the provider's `model`/`modelSecretKey`, the built-in default.""" + if sb.inference_values: + return sb.inference_values["model"] + return sb.model or (provider.model if provider else "") or default + + +def sandbox_endpoint(sb, provider): + """The endpoint the agent is onboarded against: `inference.baseUrl`, else + the provider's own base URL ("" = the protocol's native API).""" + if sb.inference_values: + return sb.inference_values["baseUrl"] + return provider.base_url if provider else "" + + +def openclaw_inference_settings(sb, provider): + """Pure onboarding choices; transports and endpoint approval stay in adapters. + + Resolved inference inputs take precedence over legacy fields. An explicit + endpoint uses custom onboarding even when its URL is a native API. A legacy + base URL reference also keeps custom onboarding when its resolved value is + absent, preserving the VM installer's compatibility behavior. + """ + model = sandbox_model(sb, provider, "nvidia/nemotron-3.5-lightning-30b-a3b") + endpoint = sandbox_endpoint(sb, provider) + protocol = sandbox_protocol(sb, provider) + if sb.inference_values: + base_url = (endpoint or NATIVE_BASE_URLS.get(provider.type, "") + or PROTOCOL_NATIVE_BASE_URLS.get(protocol, "")) + native = (not endpoint and provider.type in OPENCLAW_AUTH_CHOICES + and PROVIDER_PROTOCOLS.get(provider.type) == protocol) + else: + base_url = endpoint or NATIVE_BASE_URLS.get(provider.type, "") + native = not (endpoint or provider.base_url_secret_key) + return {"model": model, "base_url": base_url, "protocol": protocol, + "auth_choice": OPENCLAW_AUTH_CHOICES.get(provider.type) if native else None} + + +def openclaw_managed_provider_id(kind): + """Namespace explicit model catalogs away from stock catalog migrations. + + This is an OpenClaw configuration ID only. The BOM provider type, approved + endpoint/protocol and OpenShell credential binding retain their identity. + Native onboarding in the VM adapter remains unchanged. + """ + if kind not in {"openai", "anthropic", "nvidia", "custom-inference"}: + raise InstallerError("unsupported managed OpenClaw provider") + return "saw-" + kind + + +def resolve_inference(ws, sb, read_value, validate_endpoint, forbidden=()): + """A sandbox's `inference:` resolved: providerRef, protocol, baseUrl + (validated, "" = the protocol's native API), model, timeoutSeconds.""" + spec = sb.inference + where = f"sandbox '{sb.name}' in workspace '{ws.name}'" + provider = next((p for p in ws.providers if p.enabled and p.name == spec["providerRef"]), None) + out = {"providerRef": spec["providerRef"], + "protocol": inference_protocol(spec, provider.type if provider else "")} + out["model"] = resolve_bom_input(spec["model"], read_value, f"{where}: inference model", forbidden) + if not out["model"]: + raise InstallerError(f"{where}: inference model is empty") + url = resolve_bom_input(spec["baseUrl"], read_value, f"{where}: inference baseUrl", forbidden) \ + if "baseUrl" in spec else "" + if url: + try: + url = check_base_url(url) + except ValueError as exc: + raise InstallerError(f"{where}: inference baseUrl: {exc}") from None + validate_endpoint(url, spec, provider, where) + out["baseUrl"] = url + timeout = resolve_bom_input(spec["timeoutSeconds"], read_value, + f"{where}: inference timeoutSeconds", forbidden) if "timeoutSeconds" in spec else "" + if timeout and (not str(timeout).isdigit() or int(timeout) < 0): + raise InstallerError(f"{where}: inference timeoutSeconds must be a non-negative integer") + out["timeoutSeconds"] = int(timeout) if timeout else 0 + if "models" in spec: + out["models"] = {} + for name, src in spec["models"].items(): + attached = next((p for p in ws.providers if p.enabled and p.name == name + and name in sb.providers), None) + if attached is None: + continue + if attached.base_url or attached.base_url_secret_key: + raise InstallerError(f"{where}: inference models '{name}' needs a native API") + model = resolve_bom_input(src, read_value, f"{where}: inference models '{name}'", forbidden) + if not model: + raise InstallerError(f"{where}: inference models '{name}' is empty") + try: + check_env_value(model) + except ValueError as exc: + raise InstallerError(f"{where}: inference models '{name}': {exc}") from None + out["models"][name] = model + for key in ("model", "baseUrl"): + try: + check_env_value(out[key]) + except ValueError as exc: + raise InstallerError(f"{where}: inference {key}: {exc}") from None + return out + + +def sandbox_inference_problems(sb, provider_types, declared_provider_types=None): + """What is wrong with a sandbox's `inference:` block ([] when fine). + `provider_types` maps the workspace's enabled provider names to their + types.""" + spec = sb.inference + provider_names = provider_types + if not isinstance(spec, dict): + return ["inference must be a mapping"] + out = [] + unknown = sorted(set(spec) - INFERENCE_KEYS) + if unknown: + out.append(f"inference has unknown keys {', '.join(unknown)}") + ref = spec.get("providerRef") + if not ref: + out.append("inference needs providerRef (the provider whose credential it uses)") + elif ref not in provider_names: + out.append(f"inference providerRef '{ref}' is not an enabled provider in this workspace") + elif ref not in sb.providers: + out.append(f"inference providerRef '{ref}' is not in the sandbox's providers") + protocol = inference_protocol(spec, provider_types.get(ref or "")) + if "protocol" in spec and spec["protocol"] not in INFERENCE_PROTOCOLS: + out.append(f"inference protocol must be one of {', '.join(sorted(INFERENCE_PROTOCOLS))}") + elif not protocol and ref in provider_names: + out.append(f"inference needs protocol ({' | '.join(sorted(INFERENCE_PROTOCOLS))}): " + f"provider type '{provider_types[ref]}' does not imply one") + if "model" not in spec: + out.append("inference needs model") + model = spec.get("model") + if model == "" or (isinstance(model, dict) and not model.get("default") + and not model.get("required", "default" not in model)): + out.append("inference model must be non-empty: use a non-empty default or required: true") + for key in ("baseUrl", "model"): + if key in spec: + out += [f"inference {key} {p}" for p in input_problems(spec[key])] + models = spec.get("models", {}) + if not isinstance(models, dict): + out.append("inference models must be a mapping of provider names to model inputs") + else: + declared = declared_provider_types if declared_provider_types is not None else provider_types + if models and sb.type != "openclaw": + out.append("inference models is supported only for openclaw sandboxes") + for name, model in models.items(): + if name not in declared: + out.append(f"inference models names unknown provider '{name}'") + elif declared[name] not in ("openai", "anthropic"): + out.append(f"inference models '{name}' needs a native OpenAI or Anthropic provider") + out += [f"inference models '{name}' {p}" for p in input_problems(model)] + if model == "" or (isinstance(model, dict) and not model.get("default") + and not model.get("required", "default" not in model)): + out.append(f"inference models '{name}' must be non-empty") + if "timeoutSeconds" in spec: + out += [f"inference timeoutSeconds {p}" + for p in input_problems(spec["timeoutSeconds"], (str, int))] + return out + + +def harness_from_plan(data): + raw = data.get("harness") or {} + bundles = {} + for name, b in (raw.get("bundles") or {}).items(): + bundles[name] = HarnessBundle( + name=b["name"], agent=b["agent"], digest=b["digest"], files=b["files"]) + return {"bundles": bundles} + + +def profiles_from_plan(data): + profiles = [] + for p in data["profiles"]: + workspaces = [] + for w in p["workspaces"]: + workspaces.append(Workspace( + name=w["name"], enabled=w["enabled"], description=w["description"], + providers=[Provider(**x) for x in w["providers"]], + sandboxes=[Sandbox(**x) for x in w["sandboxes"]])) + profiles.append(Profile(name=p["name"], workspaces=workspaces)) + return profiles + + +__all__ = ( + 'API_VERSION', 'NAME_RE', 'SECRET_NAME_RE', 'SECRET_KEY_RE', 'OPENSHELL_NAME_LIMIT', + 'PROVIDER_CRED_MAP', 'NEMOCLAW_PROVIDER_MAP', 'nemoclaw_provider', 'profile_credential_envs', + 'profile_credentials', 'profile_all_credential_envs', 'profile_credential_env', 'credential_env_name', + 'SANDBOX_TYPES', 'BASE_URL_CONFIG_KEYS', 'InstallerError', 'Provider', 'Sandbox', 'Workspace', + 'Profile', 'HarnessBundle', 'read_profile_files', '_yaml', 'parse_profiles', 'enabled_workspaces', + 'validate_profiles', 'ENV_NAME_RE', 'CREDENTIAL_NAME_RE', 'ENV_VALUE_LIMIT', 'SANDBOX_ENV_KEYS', + 'INPUT_KEYS', 'secret_ref', 'secret_ref_problems', 'INFERENCE_KEYS', 'INFERENCE_PROTOCOLS', + 'PROVIDER_PROTOCOLS', 'check_env_value', 'sandbox_env_problems', 'input_problems', + 'check_base_url', 'resolve_bom_input', 'resolve_inference', 'inference_protocol', 'sandbox_inference_problems', 'harness_from_plan', 'profiles_from_plan', + 'validate_profile_document', 'apply_workspace_sequence', 'select_inference_provider', + 'NATIVE_BASE_URLS', 'OPENCLAW_AUTH_CHOICES', 'PROTOCOL_NATIVE_BASE_URLS', + 'sandbox_protocol', 'sandbox_model', 'sandbox_endpoint', 'openclaw_inference_settings', + 'openclaw_managed_provider_id', +) + + +def tree_digest(files): + """Shared contract: sha256 over sorted "\\0\\n" lines. + + `files` maps a POSIX relative path to file bytes. Helm computes the same + value at render time; templates/configmap-bom.yaml must agree. + """ + digest = hashlib.sha256() + for rel in sorted(files): + digest.update(f"{rel}\x00{hashlib.sha256(files[rel]).hexdigest()}\n".encode()) + return "sha256:" + digest.hexdigest() + + +_TOOL_GATE_PACKAGE = """\ +{ + "name": "saw-tool-gate", + "version": "0.1.0", + "private": true, + "type": "module", + "openclaw": { + "extensions": ["./index.js"], + "runtimeExtensions": ["./index.js"] + } +} +""" + +_TOOL_GATE_MANIFEST = """\ +{ + "id": "saw-tool-gate", + "name": "SAW tool gate", + "description": "Audit every tool call with the signed-in user and current task, and require approval before consequential tools run.", + "activation": {"onStartup": true}, + "configSchema": {"type": "object", "additionalProperties": false} +} +""" + + +def tool_gate_identity(owner, subject, workspace, sandbox): + if not isinstance(owner, str) or not owner.strip(): + raise InstallerError(f"sandbox '{sandbox}': dashboard.owner is required for the tool gate") + if not isinstance(subject, str) or not subject.strip(): + raise InstallerError(f"sandbox '{sandbox}': ownerSubject is required for the tool gate") + return {"username": owner, "subject": subject, + "sandbox": sandbox, "workspace": workspace} + + +def tool_gate_bundle(here, identity, receipt_key=None): + """Build the identical policy, identity and OpenClaw extension for both runtimes.""" + names = ("tool-actions.yaml", "tool-gate.mjs", "tool-gate-plugin.mjs") + missing = [name for name in names if not (here / name).is_file()] + if missing: + raise InstallerError("tool gate files missing from the installer disk: " + ", ".join(missing)) + policy = (here / "tool-actions.yaml").read_bytes() + policy_hash = "sha256:" + hashlib.sha256(policy).hexdigest() + plugin = (here / "tool-gate-plugin.mjs").read_bytes().replace( + b"__SAW_POLICY_HASH__", policy_hash.encode()) + if receipt_key is not None: + if not isinstance(receipt_key, str) or not re.fullmatch(r"MCowBQYDK2VwAyEA[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=", receipt_key): + raise InstallerError("audit writer requires a canonical Ed25519 public key") + plugin = plugin.replace(b"__SAW_AUDIT_KEY__", receipt_key.encode("ascii")) + tree = { + "tool-actions.yaml": (policy, False), + "identity.json": ((json.dumps(identity, sort_keys=True) + "\n").encode(), False), + "plugin/package.json": (_TOOL_GATE_PACKAGE.encode(), False), + "plugin/openclaw.plugin.json": (_TOOL_GATE_MANIFEST.encode(), False), + "plugin/index.js": (plugin, False), + "plugin/tool-gate.mjs": ((here / "tool-gate.mjs").read_bytes(), False), + } + digest = tree_digest({key: data for key, (data, _) in tree.items()}) + source = f"gate:{digest}" + return tree, source, digest, policy_hash + + +# Inline harness parsing and OpenClaw registration are shared by both installers. +HARNESS_MOUNT = "/sandbox/harness" + + +# mcp.json env/header value: exactly ${VAR}, or Authorization-style +# "Bearer ${VAR}". Any other prefix could smuggle a literal secret. +# A stdio arg naming a credential-ish flag with a literal value, e.g. +# "--api-key=sk-live-...". The "--flag value" two-token form is not covered. +MCP_PLACEHOLDER_ONLY = re.compile( + r"^(?:\$\{[A-Za-z_][A-Za-z0-9_]*\}|Bearer \$\{[A-Za-z_][A-Za-z0-9_]*\})$", + re.IGNORECASE) + + +MCP_ARG_CREDENTIAL_RE = re.compile( + r"^--?[\w-]*(?:key|token|secret|password|passwd|auth)[\w-]*=(?P.*)$", + re.IGNORECASE) + + +def safe_rel_path(rel, where): + """A bundle-relative POSIX path: no absolute, no empty or '.'/'..' segment.""" + parts = rel.split("/") + if not rel or rel.startswith("/") or any(part in ("", ".", "..") for part in parts): + raise InstallerError(f"{where}: unsafe path {rel!r}") + return rel + + +def describe_harness_tree(files, inline=False, *, warn=lambda _: None): + """What OpenClaw has to be pointed at, and what governance must allow. + + Reads harness.yaml (metadata, spec.agent, and the governance entries in + spec.plugins / spec.mcpServers) and mcp.json. OpenClaw never reads + harness.yaml; it is the installer's view of the bundle. Returns: + name, agent + skills the bundle has skills/ + agentPluginsBundle plugin.json at the root (OpenClaw loads skills/ and + mcp.json from the bundle root) + mcp the bundle has mcp.json + mcpServers server names declared in mcp.json + pluginDirs native plugins under plugins/ + governance [{kind, name, governanceProfile, hosts}] to check + against the gateway's catalog and the sandbox's + providers + + Keys never travel in a bundle. A plugin or MCP server that calls a + service names the provider profile (governanceProfile) that governs it; + the sandbox must have a provider of that type, which gives the process + the profile's env var holding a placeholder, and the sandbox's egress + proxy puts the real key in only for the profile's endpoints and + binaries. `inline` bundles come from a ConfigMap, which keeps no file + modes, so a stdio server cannot run a bundled file directly. + """ + doc = _yaml(files["harness.yaml"][0].decode("utf-8"), "harness.yaml") + spec = doc.get("spec") or {} + name = (doc.get("metadata") or {}).get("name", "") + mcp_server_names = [] + mcp_configs = {} + if "mcp.json" in files and "plugin.json" not in files: + raise InstallerError( + f"harness '{name}': mcp.json with no plugin.json at the bundle root; OpenClaw only " + "reads a bundle's mcp.json when the bundle root is on plugins.load.paths, which " + "openclaw_harness_config only sets for an Agent Plugins bundle. Add a plugin.json " + "at the bundle root") + plugin_dirs = sorted({rel.split("/")[1] for rel in files + if rel.startswith("plugins/") and rel.count("/") >= 2}) + declared_plugins = {item.get("name") for item in spec.get("plugins") or []} + for plugin in plugin_dirs: + if plugin not in declared_plugins: + warn(f"WARN: harness '{name}': plugin '{plugin}' has no entry in harness.yaml " + "spec.plugins; its egress, if any, is enforced at runtime by the sandbox " + "proxy, not checked here") + governance = [] + for item in spec.get("plugins") or []: + if item.get("governanceProfile"): + governance.append({"kind": "plugin", "name": item.get("name", ""), + "governanceProfile": item["governanceProfile"], "hosts": []}) + declared = {m.get("name"): m for m in spec.get("mcpServers") or []} + for server, decl in sorted(declared.items()): + stale = sorted(k for k in ("credentialSecret", "credentialSecretKey", "credentialEnvVar") + if k in decl) + if stale: + raise InstallerError( + f"harness '{name}': mcp server '{server}' sets {', '.join(stale)}; keys no longer " + "reach the sandbox that way. Set governanceProfile to the provider type whose " + "key the server needs and give the sandbox a provider of that type: the server " + "reads a placeholder from the provider's env var (for example BRAVE_API_KEY) " + "and the sandbox's egress proxy adds the real key") + if "mcp.json" in files: + try: + servers = json.loads(files["mcp.json"][0]).get("mcpServers") or {} + except ValueError as exc: + raise InstallerError(f"harness '{name}': mcp.json is not valid JSON ({exc})") from None + mcp_server_names = sorted(servers) + mcp_configs = {k: v for k, v in servers.items() if isinstance(v, dict)} + for server, conf in sorted(servers.items()): + if not isinstance(conf, dict) or conf.get("type") not in ("stdio", "streamable-http", "sse"): + raise InstallerError( + f"harness '{name}': mcp.json server '{server}' needs \"type\": \"stdio\", " + "\"streamable-http\" or \"sse\" (OpenClaw ignores it otherwise)") + for field in ("env", "headers"): + for key, value in (conf.get(field) or {}).items(): + if not isinstance(value, str) or not MCP_PLACEHOLDER_ONLY.match(value): + raise InstallerError( + f"harness '{name}': mcp.json server '{server}' {field} '{key}' is not " + "a bare ${VAR} placeholder (e.g. \"Bearer ${VAR}\"); a literal secret " + "must not ship in a bundle's mcp.json, the ConfigMap or the image") + for arg in conf.get("args") or []: + match = isinstance(arg, str) and MCP_ARG_CREDENTIAL_RE.match(arg) + if match and not MCP_PLACEHOLDER_ONLY.match(match.group("value")): + raise InstallerError( + f"harness '{name}': mcp.json server '{server}' arg {arg!r} looks like a " + "credential flag with a literal value; use --flag=${VAR} instead") + if isinstance(conf.get("url"), str): + creds = urlsplit(conf["url"]) + if creds.username or creds.password: + raise InstallerError( + f"harness '{name}': mcp.json server '{server}' url has a literal " + "credential in it (user:pass@host); a literal secret must not ship " + "in a bundle's mcp.json, the ConfigMap or the image") + profile = (declared.get(server) or {}).get("governanceProfile", "") + if conf["type"] == "stdio": + command = conf.get("command") + if inline and isinstance(command, str) and "${PLUGIN_ROOT}" in command: + raise InstallerError( + f"harness '{name}': mcp server '{server}' runs {command!r}, a file in the " + "bundle; an inline bundle keeps no file modes, so run it through its " + "interpreter (for example command: node, args: [${PLUGIN_ROOT}/...]) " + "or ship the bundle as an image") + # A stdio server runs inside the sandbox, under its policy. It + # needs a governanceProfile only when it calls a service. + if profile: + governance.append({"kind": "MCP server", "name": server, + "governanceProfile": profile, "hosts": []}) + else: + warn(f"WARN: harness '{name}': stdio MCP server '{server}' has no " + "governanceProfile; its egress, if any, is enforced at runtime by " + "the sandbox proxy, not checked here") + continue + host = urlsplit(conf.get("url") or "").hostname + if not host: + raise InstallerError(f"harness '{name}': mcp.json server '{server}' has no url host") + if not profile: + raise InstallerError( + f"harness '{name}': remote MCP server '{server}' ({host}) needs a " + "governanceProfile in harness.yaml spec.mcpServers") + governance.append({"kind": "MCP server", "name": server, + "governanceProfile": profile, "hosts": [host]}) + bundle_id = name + if "plugin.json" in files: + try: + bundle_id = json.loads(files["plugin.json"][0].decode("utf-8")).get("name") or name + except (ValueError, AttributeError): + pass + return { + "name": name, + "bundleId": bundle_id, + "agent": spec.get("agent", "openclaw"), + "skills": any(rel.startswith("skills/") for rel in files), + "agentPluginsBundle": "plugin.json" in files, + "mcp": "mcp.json" in files, + "mcpServers": mcp_server_names, + "mcpConfigs": mcp_configs, + "pluginDirs": plugin_dirs, + "governance": governance, + } + + +def openclaw_harness_config(info): + """OpenClaw config that makes it load the mounted bundle. + + An Agent Plugins bundle (plugin.json at the root) brings its skills/ and + mcp.json (MCP servers) with it, so the bundle root goes on + plugins.load.paths. Otherwise skills/ is an extra skill directory. + plugins/ holds native OpenClaw plugins (tool code); OpenClaw discovers + every plugin under it from the one parent path, so adding or removing a + plugin in the bundle needs no config change (checked live on OpenClaw + 2026.9.5). + """ + config = {} + paths = [] + if info["agentPluginsBundle"]: + paths.append(HARNESS_MOUNT) + elif info["skills"]: + config["skills.load.extraDirs"] = [f"{HARNESS_MOUNT}/skills"] + if info["pluginDirs"]: + paths.append(f"{HARNESS_MOUNT}/plugins") + if paths: + config["plugins.load.paths"] = paths + return config + + +BUNDLE_ROOT_PLACEHOLDERS = ("${CLAUDE_PLUGIN_ROOT}", "${PLUGIN_ROOT}") + + +MCP_SERVER_NAME_RE = re.compile(r"^[A-Za-z0-9_-]+$") + + +# OpenClaw (2026.9.x) expands only ${PLUGIN_ROOT}, ${PLUGIN_DATA} and +# ${CLAUDE_PLUGIN_ROOT} in a bundle's mcp.json; any other ${VAR} reaches the +# server as literal text, and a stdio server inherits only HOME, LOGNAME, +# PATH, SHELL, TERM and USER. So a bundle server never sees its provider's +# placeholder (Slack answers invalid_auth) or the sandbox's CA settings (TLS +# through the egress proxy fails). openclaw.json does expand ${VAR} from the +# gateway's environment, which has both, and a server defined under +# mcp.servers replaces the bundle's server of the same name (found live: the +# daily-briefing Slack reader works only this way). +# The CA settings OpenShell gives a sandbox's processes when it has the proxy +# CA (openshell-sandbox child_env::tls_env_vars). Only the ones the sandbox +# really has are passed on: OpenClaw leaves an unset ${VAR} as literal text, +# and REQUESTS_CA_BUNDLE='${REQUESTS_CA_BUNDLE}' breaks every HTTPS call. +MCP_TRUST_ENV = ("NODE_EXTRA_CA_CERTS", "DENO_CERT", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", "GIT_SSL_CAINFO") + + +def harness_mcp_servers(info, trust_env=(), *, warn=lambda _: None): + """{name: mcp.servers entry} re-declaring the bundle's MCP servers in + openclaw.json, where OpenClaw expands ${VAR}. Stdio servers also get the + CA settings named in trust_env (the ones the sandbox has), so they trust + the egress proxy. Paths resolve as OpenClaw resolves them for a bundle + (root: the harness mount).""" + if info is None or not info.get("agentPluginsBundle"): + return {} + root = HARNESS_MOUNT + data = f"/sandbox/.openclaw/saw-plugin-data/{info['bundleId']}" + + def expand(value): + if not isinstance(value, str): + return value + for placeholder in BUNDLE_ROOT_PLACEHOLDERS: + value = value.replace(placeholder, root) + return value.replace("${PLUGIN_DATA}", data) + + out = {} + for name, conf in sorted((info.get("mcpConfigs") or {}).items()): + if not MCP_SERVER_NAME_RE.match(name): + warn(f"WARN: harness '{info['name']}': MCP server name {name!r} cannot be " + "re-declared in openclaw.json; it keeps the bundle's literal env") + continue + kind = conf.get("type") + if kind == "stdio": + command = conf.get("command") + if isinstance(command, str) and command.startswith("./"): + command = f"{root}/{command[2:]}" + entry = {"command": expand(command), + "args": [expand(a) for a in conf.get("args") or []], + "env": {"PLUGIN_ROOT": root, "PLUGIN_DATA": data, + **{var: f"${{{var}}}" for var in trust_env}, + **{k: expand(v) for k, v in (conf.get("env") or {}).items()}}} + if conf.get("cwd"): + entry["cwd"] = expand(conf["cwd"]) + elif kind in ("streamable-http", "sse"): + if not conf.get("headers"): + continue # nothing for OpenClaw to expand: the bundle's own entry works + entry = {"url": expand(conf.get("url")), "transport": kind, + "headers": {k: expand(v) for k, v in conf["headers"].items()}} + else: + continue + out[name] = entry + return out + + +def harness_plugin_ids(info): + """The OpenClaw plugin ids the harness brings: the bundle's (plugin.json + name) and each native plugin under plugins/.""" + ids = [info["bundleId"]] if info["agentPluginsBundle"] else [] + return ids + list(info["pluginDirs"]) + + +def parse_harness_files(files): + """Build bundles from flat ConfigMap keys harness____, plus + one harness____map key per bundle mapping each hash to its + relpath (content-addressed: the key no longer encodes the path, so it + never hits the iso9660/Joliet 64-character filename limit regardless of + how deep or long a bundle's paths are; see templates/configmap-bom.yaml). + + The digest is computed, never authored here: the pin lives in + sandbox.yaml harnessRef, outside the hashed tree. Relpaths from the map + get the same .. / empty / absolute checks as read_harness_tar. + """ + raw_by_bundle, maps = {}, {} + for key, raw in files.items(): + parts = key.split("__") + if len(parts) != 3 or parts[0] != "harness": + raise InstallerError(f"unexpected harness file name: {key}") + bundle, token = parts[1], parts[2] + if token == "map": + try: + maps[bundle] = json.loads(raw.decode("utf-8")) + except (ValueError, UnicodeDecodeError) as exc: + raise InstallerError( + f"harness bundle {bundle!r}: map is not valid JSON ({exc})") from None + else: + raw_by_bundle.setdefault(bundle, {})[token] = raw + bundles = {} + for name in sorted(raw_by_bundle): + path_map = maps.get(name) + if path_map is None: + raise InstallerError(f"harness bundle {name!r} has no path map (harness__{name}__map)") + tree = {} + for hash_, raw in raw_by_bundle[name].items(): + rel = path_map.get(hash_) + if rel is None: + raise InstallerError( + f"harness bundle {name!r}: key 'harness__{name}__{hash_}' is not in its path map") + tree[safe_rel_path(rel, f"harness bundle {name!r} path map")] = raw + if "harness.yaml" not in tree: + raise InstallerError(f"harness bundle {name!r} has no harness.yaml") + doc = _yaml(tree["harness.yaml"].decode("utf-8"), f"harness/{name}/harness.yaml") + spec = doc.get("spec") or {} + bundles[name] = HarnessBundle( + name=(doc.get("metadata") or {}).get("name", name), + agent=spec.get("agent", "openclaw"), + digest=tree_digest(tree), + files={rel: base64.b64encode(raw).decode() for rel, raw in tree.items()}, + ) + return bundles + + +def pinned_bundle(sb, bundles): + """The inline bundle a sandbox's harnessRef.name points at. The bundle + and the pin ship in the same ConfigMap, so harnessRef.digest is + optional; when it is set it must match.""" + ref = sb.harness_ref or {} + bundle = bundles.get(ref.get("name")) + if bundle is None: + raise InstallerError(f"sandbox '{sb.name}' references unknown harness bundle " + f"'{ref.get('name')}'") + want = ref.get("digest") or "" + if want and want != bundle.digest: + raise InstallerError( + f"harnessRef digest mismatch for sandbox '{sb.name}': sandbox.yaml says {want}, " + f"the bundle hashes to {bundle.digest}") + return bundle + + +def check_harness_index(bundles, index): + """harness-index.yaml holds the digest Helm computed for each shipped + bundle; the installer recomputes it from the files it read. A mismatch + means the chart and the installer disagree on the tree_digest contract + or on the files (an encoding change on the way), so nothing is trusted.""" + listed = (index or {}).get("bundles") or {} + for name, digest in sorted(listed.items()): + bundle = bundles.get(name) + if bundle is None: + raise InstallerError(f"harness-index.yaml lists bundle '{name}', but no files for it " + "were shipped") + if digest != bundle.digest: + raise InstallerError( + f"harness bundle '{name}': the chart computed {digest}, the installer " + f"{bundle.digest}; the files changed on the way or the tree_digest contract " + "differs between templates/configmap-bom.yaml and apply_bom.py") + for name in sorted(set(bundles) - set(listed)): + if listed or index: + raise InstallerError(f"harness bundle '{name}' is missing from harness-index.yaml") + +__all__ += ('BUNDLE_ROOT_PLACEHOLDERS', 'HARNESS_MOUNT', 'MCP_ARG_CREDENTIAL_RE', 'MCP_PLACEHOLDER_ONLY', 'MCP_SERVER_NAME_RE', 'check_harness_index', 'harness_plugin_ids', 'openclaw_harness_config', 'parse_harness_files', 'pinned_bundle', 'safe_rel_path') + + +def host_allowed(host, patterns): + """A profile endpoint host may be a glob (*-aiplatform.googleapis.com); + ports are not part of the match. Each `*` is one DNS label (no dots).""" + host = (host or "").split(":")[0].lower() + for pattern in patterns: + regex = re.escape((pattern or "").split(":")[0].lower()).replace(r"\*", r"[^.]*") + if re.fullmatch(regex, host): + return True + return False + + +def check_harness_governance(ws, sb, info, catalog, attached): + for item in info["governance"]: + profile = item["governanceProfile"] + what = f"harness '{info['name']}' {item['kind']} '{item['name']}'" + if profile not in catalog: + raise InstallerError( + f"{what} names governanceProfile '{profile}', which the gateway does not " + f"serve in workspace '{ws.name}'; refusing to fill the harness of sandbox " + f"'{sb.name}'") + if profile not in attached: + raise InstallerError( + f"{what} names governanceProfile '{profile}', but sandbox '{sb.name}' has no " + f"provider of type '{profile}'; add one to the workspace's providers.yaml and " + "to the sandbox's providers (its endpoints and key reach the sandbox only " + "through a provider)") + for host in item["hosts"]: + if not host_allowed(host, catalog[profile]): + allowed = ", ".join(sorted(catalog[profile])) or "none" + raise InstallerError( + f"{what} reaches {host}, which governanceProfile '{profile}' does not " + f"allow (allowed: {allowed}); refusing to fill the harness of " + f"sandbox '{sb.name}'") + + +def mcp_registration_failures(output, declared): + """Inspect reports configuration registration, never MCP process readiness. + + Kept pure so the E2E script can apply the same checks to inspection JSON. + Names in descriptions or diagnostics are not evidence of registration. + """ + try: + report = json.loads(output) + plugin = report["plugin"] + servers = report["mcpServers"] + diagnostics = report.get("diagnostics") or [] + if not isinstance(plugin, dict) or not isinstance(servers, list) or not isinstance(diagnostics, list): + raise ValueError("invalid inspection shape") + if plugin.get("status") != "loaded" or plugin.get("enabled") is False or plugin.get("error"): + return ["OpenClaw bundle runtime inspection reports an error or disabled bundle"] + if any(isinstance(d, dict) and d.get("level") == "error" for d in diagnostics): + return ["OpenClaw bundle runtime inspection reports error diagnostics"] + registered = {entry["name"] for entry in servers + if isinstance(entry, dict) and isinstance(entry.get("name"), str) + and not entry.get("unsupported")} + except (ValueError, TypeError, KeyError): + return ["could not parse OpenClaw MCP registration inspection JSON"] + missing = sorted(set(declared) - registered) + return ([f"OpenClaw does not list MCP server(s) {', '.join(missing)} as registered"] + if missing else []) + + +def enable_harness_plugin(plugin_id, environment=""): + """Grant bundle capability consent using the stock CLI, including older versions.""" + command = (environment + " openclaw plugins enable " + shlex.quote(plugin_id)).lstrip() + return f"{command} --accept-capabilities || {command}" diff --git a/charts/openshell-saw/files/installer/saw_ui_limit.py b/charts/openshell-saw/files/installer/saw_ui_limit.py new file mode 100644 index 000000000..c5c29d009 --- /dev/null +++ b/charts/openshell-saw/files/installer/saw_ui_limit.py @@ -0,0 +1,369 @@ +"""Caps the connections to `openshell forward service` (SAW sandbox UI). + +OpenShell allows 20 concurrent forward connections per sandbox and closes +the rest (RESOURCE_EXHAUSTED, "sandbox SSH connection limit reached", +NVIDIA/OpenShell#3494). A browser loading the OpenClaw control UI through +the route opens more than that at once, and the requests that lost came +back as 502/504 after 30 s. This TCP relay sits between oauth2-proxy and the +forward: + +- at most MAX connections reach the forward; the others wait for a free one; +- while some wait, an HTTP keep-alive connection that has been idle for + IDLE_PREEMPT seconds after an answer is closed to make room (an HTTP + client opens a new one); WebSocket connections are never closed; +- a connection the forward closes or resets before answering is retried; +- a GET that gets no answer at all within ANSWER_WAIT is sent again on new + connections, and the first answer wins. + + python3 saw_ui_limit.py LISTEN_PORT UPSTREAM_PORT [MAX] [--connect-only] + +The container relay uses --connect-only: reconnect only before sending bytes, +never replay an HTTP request or WebSocket frame. The stable listener absorbs +brief stock-CLI restarts. Established WebSockets still close when upstream +authentication expires; clients must reconnect normally. +""" +import asyncio +import os +import re +from pathlib import Path +import sys +import time + + +def log(message): + print(message, file=sys.stderr, flush=True) + +RETRIES = 8 +BUFFER_LIMIT = 1 << 20 +ANSWER_WAIT = 2.0 # seconds to wait for a first answer byte before giving up retries +IDLE_PREEMPT = 2.0 # seconds an answered keep-alive connection may hold a slot others wait for +HEDGES = 2 # extra connections for a repeatable request that gets no answer +HEDGE_TOTAL = 25.0 # seconds before giving up on such a request (the router allows 30) +# For a request that is not safe to send again and lost its answer. +_AMBIGUOUS_BODY = b"The sandbox closed the connection; the request may have been processed.\n" +AMBIGUOUS_ANSWER = (b"HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\n" + b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(_AMBIGUOUS_BODY)) + _AMBIGUOUS_BODY + + +class Conn: + """One client connection holding (or waiting for) a slot.""" + + def __init__(self, websocket): + self.websocket = websocket + self.last = time.monotonic() + self.answered = False # the forward spoke last: no request in flight + self.writers = [] + + def touch(self, answered): + self.last = time.monotonic() + self.answered = answered + + def idle_for(self): + return time.monotonic() - self.last + + def close(self): + for w in self.writers: + try: + w.transport.abort() + except Exception: + pass + + +class Slots: + """MAX upstream connections at a time; idle keep-alive ones give way. + + A client connection holds one slot for its upstream connection. A hedge + (an extra connection for a repeatable request that got no answer) takes + a slot of its own, only when one is free, and gives it back as soon as + it is closed or becomes the connection that answered.""" + + def __init__(self, limit): + self.limit = limit + self.active = set() + self.extra = 0 # hedge connections open now + self.cond = asyncio.Condition() + + def in_use(self): + return len(self.active) + self.extra + + def try_extra(self): + """Take a slot for a hedge connection if one is free; never waits. + (No await: atomic within the event loop.)""" + if self.in_use() >= self.limit: + return False + self.extra += 1 + return True + + async def release_extra(self, count=1): + if count <= 0: + return + async with self.cond: + self.extra = max(0, self.extra - count) + self.cond.notify(count) + + def victim(self): + idle = [c for c in self.active + if not c.websocket and c.answered and c.idle_for() >= IDLE_PREEMPT] + return min(idle, key=lambda c: c.last) if idle else None + + def describe(self): + ws = sum(1 for c in self.active if c.websocket) + busy = sum(1 for c in self.active if not c.websocket and not c.answered) + idle = sorted(round(c.idle_for(), 1) for c in self.active if not c.websocket and c.answered) + return (f"{self.in_use()}/{self.limit} in use: {ws} websocket, {busy} awaiting an answer, " + f"{len(idle)} answered (idle s: {idle}), {self.extra} hedge") + + async def acquire(self, conn): + start = time.monotonic() + logged = 0.0 + async with self.cond: + while self.in_use() >= self.limit: + v = self.victim() + if v is not None: + log(f"closing a keep-alive connection idle {v.idle_for():.1f}s to make room") + self.active.discard(v) + v.close() + break + waited = time.monotonic() - start + if waited - logged >= 5: + logged = waited + log(f"waiting {waited:.0f}s for a slot; {self.describe()}") + try: + await asyncio.wait_for(self.cond.wait(), 0.25) + except asyncio.TimeoutError: + pass + self.active.add(conn) + if time.monotonic() - start >= 1: + log(f"got a slot after {time.monotonic() - start:.1f}s; {self.describe()}") + + async def release(self, conn): + async with self.cond: + if conn in self.active: + self.active.discard(conn) + self.cond.notify() + + +async def relay(reader, writer, conn, answered): + """Copy until EOF, then half-close the other side.""" + try: + while True: + data = await reader.read(65536) + if not data: + break + conn.touch(answered) + writer.write(data) + await writer.drain() + if writer.can_write_eof(): + writer.write_eof() + except (ConnectionError, OSError): + pass + + +class Ambiguous(Exception): + """The request reached the forward, but no answer came back: it may or + may not have been processed, so it is not sent again.""" + + +async def open_answering(upstream, sent, slots, retry_sent=True): + """Connect to the forward and send the client's first bytes. + + Retries when nothing was sent (the connection failed), and, for a + request that is safe to send again (replayable), when the forward + closes or resets the connection without answering: that is how it + refuses a connection past its limit. Any other request may already + have been processed by then, so it is not sent again: Ambiguous.""" + for attempt in range(RETRIES + 1): + try: + ureader, uwriter = await asyncio.open_connection("127.0.0.1", upstream) + except OSError: + await asyncio.sleep(min(1.0, 0.2 * (attempt + 1))) + continue + if not retry_sent: + try: + uwriter.write(sent) + await uwriter.drain() + except (ConnectionError, OSError): + uwriter.close() + raise Ambiguous() + # Once the connection exists, relay exactly once. Even a GET that + # loses its response must not be replayed in connect-only mode. + return ureader, uwriter, b"" + try: + uwriter.write(sent) + await uwriter.drain() + # A refused forward connection closes (or resets) at once. A + # request still being sent (a large body) gets no answer yet: + # relay it. + first = await asyncio.wait_for(ureader.read(65536), ANSWER_WAIT) + except asyncio.TimeoutError: + if not hedgeable(sent): + log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); relaying without retries") + return ureader, uwriter, b"" + return await hedge(upstream, sent, ureader, uwriter, slots) + except (ConnectionError, OSError): + first = b"" + if first: + return ureader, uwriter, first + uwriter.close() + if not replayable(sent): + raise Ambiguous() + await asyncio.sleep(min(1.0, 0.2 * (attempt + 1))) + return None, None, b"" + + +def replayable(sent): + """A complete GET/HEAD/OPTIONS request without a body (a WebSocket + upgrade included: no answer means no upgrade): sending it again cannot + repeat a change.""" + head = sent.split(b"\r\n\r\n", 1) + return len(head) == 2 and not head[1] and sent.split(b" ", 1)[0] in (b"GET", b"HEAD", b"OPTIONS") + + +def hedgeable(sent): + """A replayable request that is not a WebSocket upgrade: safe to send + again on another connection while the first is still open.""" + return replayable(sent) and b"upgrade: websocket" not in sent.lower() + + +async def hedge(upstream, sent, ureader, uwriter, slots): + """Found live: a request through the forward sometimes got no answer at + all (no error, nothing in the forward's log) and the browser saw a 504. + For a request that is safe to repeat, send it again on new connections + and keep whichever answers first. Each extra connection needs a free + slot (Slots.try_extra); without one, the request just keeps waiting on + the connections it has.""" + pending = {asyncio.ensure_future(ureader.read(65536)): (ureader, uwriter)} + extras = 0 + deadline = time.monotonic() + HEDGE_TOTAL + try: + for extra in range(HEDGES + 1): + if extra < HEDGES: + if slots.try_extra(): + extras += 1 + log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); " + f"sending it again ({extra + 1})") + try: + r, w = await asyncio.open_connection("127.0.0.1", upstream) + w.write(sent) + await w.drain() + pending[asyncio.ensure_future(r.read(65536))] = (r, w) + except (ConnectionError, OSError): + pass + else: + log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); " + "no free slot to send it again") + wait = ANSWER_WAIT * 2 if extra < HEDGES else max(0.0, deadline - time.monotonic()) + done, _ = await asyncio.wait(pending, timeout=wait, return_when=asyncio.FIRST_COMPLETED) + for task in done: + r, w = pending.pop(task) + try: + data = task.result() + except (ConnectionError, OSError): + data = b"" + if data: + for other, (_, ow) in pending.items(): + other.cancel() + ow.close() + pending.clear() + return r, w, data + w.close() + if not pending: + break + for other, (_, ow) in pending.items(): + other.cancel() + ow.close() + log(f"no answer on {extras + 1} connections ({sent[:60]!r})") + return None, None, b"" + finally: + # The connection that answered (if any) runs on the client's own + # slot; every extra one taken here is closed by now. + await slots.release_extra(extras) + + +async def relay_request(reader, proof_file, subject): + # Only the authenticated loopback proxy may supply the owner subject. + # Replace every caller-supplied proof header before sending to OpenShell. + raw = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 10) + if len(raw) > 65536: + raise ValueError("oversized browser request") + request, *headers = raw[:-4].split(b"\r\n") + checked = [] + owners = [] + websocket = False + for line in headers: + name, sep, value = line.partition(b":") + if not sep or not re.fullmatch(rb"[!#$%&'*+.^_`|~0-9a-zA-Z-]+", name): + raise ValueError("invalid browser header") + if name.lower() == b"x-forwarded-email": + owners.append(value.strip()) + if name.lower() == b"upgrade" and value.strip().lower() == b"websocket": + websocket = True + if name.lower() not in (b"x-saw-relay-identity", b"connection"): + checked.append(line) + if owners != [subject.encode("ascii")]: + raise ValueError("missing authenticated owner") + proof = Path(proof_file).read_bytes() + if not re.fullmatch(rb"[a-f0-9]{64}", proof): + raise ValueError("invalid relay proof") + checked.append(b"Connection: Upgrade" if websocket else b"Connection: close") + checked.append(b"X-Saw-Relay-Identity: " + subject.encode("ascii") + b"." + proof) + return b"\r\n".join([request, *checked]) + b"\r\n\r\n" + + +def handler(upstream, slots, retry_sent=True): + async def handle(creader, cwriter): + conn = None + try: + # The first bytes tell an HTTP request from a WebSocket upgrade, + # and are what a retry resends. + proof_file = os.environ.get("SAW_BROWSER_RELAY_PROOF_FILE") + if proof_file: + sent = await relay_request(creader, proof_file, os.environ["SAW_BROWSER_OWNER_SUBJECT"]) + else: + sent = await creader.read(65536) + if not sent: + return + conn = Conn(b"upgrade: websocket" in sent.lower()) + conn.writers.append(cwriter) + await slots.acquire(conn) + try: + ureader, uwriter, first = await open_answering(upstream, sent, slots, retry_sent) + except Ambiguous: + log("the forward closed without an answer; " + "not sending it again, it may have been processed") + cwriter.write(AMBIGUOUS_ANSWER) + await cwriter.drain() + return + if ureader is None: + log("the forward refused the connection on every retry") + return + conn.writers.append(uwriter) + conn.touch(bool(first)) + if first: + cwriter.write(first) + await cwriter.drain() + await asyncio.gather(relay(creader, uwriter, conn, False), + relay(ureader, cwriter, conn, True)) + uwriter.close() + except (ConnectionError, OSError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError, TimeoutError): + pass + finally: + if conn is not None: + await slots.release(conn) + cwriter.close() + return handle + + +async def main(listen, upstream, limit, retry_sent=True): + server = await asyncio.start_server(handler(upstream, Slots(limit), retry_sent), "127.0.0.1", listen, + limit=BUFFER_LIMIT) + async with server: + await server.serve_forever() + + +if __name__ == "__main__": + listen_port, upstream_port = int(sys.argv[1]), int(sys.argv[2]) + max_conns = int(sys.argv[3]) if len(sys.argv) > 3 else 10 + if len(sys.argv) > 5 or (len(sys.argv) == 5 and sys.argv[4] != "--connect-only"): + raise SystemExit("expected LISTEN_PORT UPSTREAM_PORT [MAX] [--connect-only]") + asyncio.run(main(listen_port, upstream_port, max_conns, retry_sent=len(sys.argv) < 5)) diff --git a/charts/openshell-saw/files/installer/tool-gate-plugin.mjs b/charts/openshell-saw/files/installer/tool-gate-plugin.mjs index 0f0bc5ac0..783e1facb 100644 --- a/charts/openshell-saw/files/installer/tool-gate-plugin.mjs +++ b/charts/openshell-saw/files/installer/tool-gate-plugin.mjs @@ -1,3 +1,5 @@ +import fs from "node:fs"; +import {createHash, createPublicKey, randomUUID, verify} from "node:crypto"; // OpenClaw before_tool_call hook. Consequential tools come from // tool-actions.yaml. This file does not list MCP servers or probe them. import { @@ -14,6 +16,43 @@ import { // the read-only gate volume. No writable node_modules link is needed. const EXPECTED_POLICY_HASH = "__SAW_POLICY_HASH__"; +// The public key is compiled into the read-only plugin by the SAW bundle builder. +function submitAuditRecord(publicKey, record) { + const directory = "/sandbox/.saw/audit"; + const id = randomUUID(); + const body = JSON.stringify(record); + const fd = fs.openSync(directory + "/" + id + ".tmp", "wx", 0o600); + try { fs.writeFileSync(fd, body); fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + fs.renameSync(directory + "/" + id + ".tmp", directory + "/" + id + ".req"); + const directoryFd = fs.openSync(directory, fs.constants.O_DIRECTORY | fs.constants.O_RDONLY); + try { fs.fsyncSync(directoryFd); } finally { fs.closeSync(directoryFd); } + const hash = createHash("sha256").update(body).digest("hex"); + const challenge = Buffer.from("saw-audit-v1:" + id + ":" + hash); + const key = createPublicKey({key:Buffer.from(publicKey, "base64"), format:"der", type:"spki"}); + const pause = new Int32Array(new SharedArrayBuffer(4)); + const receipt = directory + "/" + id + ".ack"; + const deadline = performance.now() + 5000; + while (performance.now() < deadline) { + let fd; + try { + fd = fs.openSync(receipt, fs.constants.O_RDONLY | fs.constants.O_NONBLOCK | fs.constants.O_NOFOLLOW); + if (!fs.fstatSync(fd).isFile()) throw new Error("invalid audit receipt"); + const bytes = Buffer.alloc(89); + const size = fs.readSync(fd, bytes, 0, bytes.length, 0); + if (size !== 88 || !verify(null, challenge, key, Buffer.from(bytes.toString("ascii",0,size),"base64"))) + throw new Error("untrusted audit receipt"); + fs.unlinkSync(receipt); + return; + } catch (err) { + if (err.code !== "ENOENT") throw err; + } finally { if (fd !== undefined) fs.closeSync(fd); } + Atomics.wait(pause,0,0,50); + } + throw new Error("audit writer did not acknowledge"); +} + +const receiptKey = "__SAW_AUDIT_KEY__"; + export function handleBeforeToolCall(event, ctx) { const taskFile = readJson(TASK_PATH); const result = evaluateToolCall({ @@ -26,6 +65,10 @@ export function handleBeforeToolCall(event, ctx) { sessionKey: (ctx && ctx.sessionKey) || (event && event.sessionKey) || "", sessionId: (ctx && ctx.sessionId) || (event && event.sessionId) || "", auditFile: AUDIT_PATH, + // OpenClaw catches callback exceptions and then permits the tool. End this + // executor synchronously before it can act on an unacknowledged approval. + unacknowledgedApproval: receiptKey.startsWith("MCowBQYDK2VwAyEA") ? () => process.exit(75) : undefined, + appendRecord: receiptKey.startsWith("MCowBQYDK2VwAyEA") ? record => submitAuditRecord(receiptKey, record) : undefined, }); return result.hook; } diff --git a/charts/openshell-saw/files/installer/tool-gate.mjs b/charts/openshell-saw/files/installer/tool-gate.mjs index 2d1af7833..3429a31ee 100644 --- a/charts/openshell-saw/files/installer/tool-gate.mjs +++ b/charts/openshell-saw/files/installer/tool-gate.mjs @@ -369,9 +369,10 @@ export function evaluateToolCall(input) { sessionId: input.sessionId, now: input.now, }); - if (input.auditFile) { + const append = input.appendRecord || (input.auditFile && (record => appendAudit(input.auditFile, record))); + if (append) { try { - appendAudit(input.auditFile, decided.record); + append(decided.record); } catch { const record = { ...decided.record, rule: "audit-unwritable", decision: "deny" }; process.stderr.write(`${JSON.stringify(record)}\n`); @@ -382,12 +383,15 @@ export function evaluateToolCall(input) { } } const hook = hookResult(decided, (resolution) => { - if (!input.auditFile) return; + if (!append) return; try { - appendAudit(input.auditFile, resolutionRecord(decided.record, resolution)); + append(resolutionRecord(decided.record, resolution)); } catch { const record = { ...resolutionRecord(decided.record, resolution), auditWriteFailed: true }; process.stderr.write(`${JSON.stringify(record)}\n`); + if (record.decision === "approved" && input.unacknowledgedApproval) { + input.unacknowledgedApproval(); + } } }); return { record: decided.record, hook }; diff --git a/charts/openshell-saw/templates/_governed-proxy-policy.tpl b/charts/openshell-saw/templates/_governed-proxy-policy.tpl new file mode 100644 index 000000000..39b52d032 --- /dev/null +++ b/charts/openshell-saw/templates/_governed-proxy-policy.tpl @@ -0,0 +1,52 @@ +{{/* Shared broker config for VM and container. The catalog generator mirrors this template into saw-users. */}} +{{- define "saw.governedProxyPolicy" -}} +{{- $profiles := dict -}} +{{- $refresh := dict -}} +{{- range $profile := .catalog -}} + {{- range $credential := $profile.credentials | default list -}} + {{- $spec := $credential.refresh | default dict -}} + {{- if eq ($spec.strategy | default "") "oauth2_refresh_token" -}} + {{- $keys := list -}} + {{- range $spec.material -}}{{- $keys = append $keys .name -}}{{- end -}} + {{- range list "client_id" "client_secret" "refresh_token" -}} + {{- if not (has . $keys) -}}{{- fail "OAuth refresh requires client_id, client_secret and refresh_token material" -}}{{- end -}} + {{- end -}} + {{- $_ := set $refresh $profile.id (dict "credential" $credential.name "spec" (dict "strategy" $spec.strategy "tokenURL" $spec.token_url "refreshBeforeSeconds" $spec.refresh_before_seconds "maxLifetimeSeconds" $spec.max_lifetime_seconds "material" (dict "clientID" "client_id" "clientSecret" "client_secret" "refreshToken" "refresh_token"))) -}} + {{- end -}} + {{- end -}} + {{- if $profile.credentials -}} + {{- $hosts := list -}} + {{- range $profile.endpoints -}} + {{- if and .host (not (has .host $hosts)) -}}{{- $hosts = append $hosts .host -}}{{- end -}} + {{- end -}} + {{- $_ := set $profiles $profile.id $hosts -}} + {{- end -}} +{{- end -}} +{{- if hasKey $profiles "nvidia" -}}{{- $_ := set $profiles "build" (index $profiles "nvidia") -}}{{- end -}} +{{- $bindings := list -}} +{{- $attached := .attached -}} +{{- range .bindings -}} + {{- if has .name $attached -}} + {{- if and .name (not (regexMatch "^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" .name)) -}}{{ fail (printf "invalid secret name %q" .name) }}{{- end -}} + {{- if not (regexMatch "^[A-Za-z0-9_.-]+$" (.key | default "")) -}} + {{- fail (printf "credentialProxy.secrets[%s].key is invalid" .name) -}} + {{- end -}} + {{- if not .providers -}}{{- fail (printf "credentialProxy.secrets[%s].providers is required" .name) -}}{{- end -}} + {{- if and (hasKey . "refresh") (not (kindIs "bool" .refresh)) -}} + {{- fail "credentialProxy.secrets[].refresh must be a boolean" -}} + {{- end -}} + {{- $binding := dict "secret" .name "key" .key "providers" .providers -}} + {{- if .refresh -}} + {{- if ne (len .providers) 1 -}}{{- fail "refresh binding requires exactly one provider type" -}}{{- end -}} + {{- $type := first .providers -}} + {{- if not (hasKey $refresh $type) -}}{{- fail (printf "provider %s has no OAuth refresh spec" $type) -}}{{- end -}} + {{- $entry := index $refresh $type -}} + {{- if ne .key $entry.credential -}}{{- fail "refresh binding key must match the profile credential" -}}{{- end -}} + {{- $_ := set $binding "refresh" $entry.spec -}} + {{- end -}} + {{- $bindings = append $bindings $binding -}} + {{- end -}} +{{- end -}} +{{- $policy := dict "markerPrefix" "saw-cred" "allowHosts" .allowHosts "profiles" $profiles "bindings" $bindings -}} +{{ $policy | toJson }} +{{- end -}} diff --git a/charts/openshell-saw/templates/_helpers.tpl b/charts/openshell-saw/templates/_helpers.tpl index 2ed4335a0..1dab8fef8 100644 --- a/charts/openshell-saw/templates/_helpers.tpl +++ b/charts/openshell-saw/templates/_helpers.tpl @@ -696,3 +696,22 @@ key-less -meta copies; the issuer CA bundle and registry pull token as is. */}} {{- end -}} {{- end -}} {{- end -}} + +{{/* Never let a requested but unimplemented backend fall through to the VM. */}} +{{- define "openshell-sandbox.validateDeploymentMode" -}} +{{- $deployment := .Values.deployment -}} +{{- if not (kindIs "map" $deployment) -}} +{{- fail "deployment must be an object with a mode" -}} +{{- end -}} +{{- range $key, $_ := $deployment -}} +{{- if ne $key "mode" -}}{{- fail (printf "unknown deployment field %q" $key) -}}{{- end -}} +{{- end -}} +{{- $mode := get $deployment "mode" -}} +{{- if not (kindIs "string" $mode) -}}{{- fail "deployment.mode must be a string" -}}{{- end -}} +{{- if not (has $mode (list "vm" "container")) -}} +{{- fail "deployment.mode must be one of vm, container" -}} +{{- end -}} +{{- if ne $mode "vm" -}} +{{- fail (printf "deployment.mode=%s is not implemented in this revision; no VM fallback (see docs/container-workspaces.md)" $mode) -}} +{{- end -}} +{{- end -}} diff --git a/charts/openshell-saw/templates/configmap-installer.yaml b/charts/openshell-saw/templates/configmap-installer.yaml index 2c9b9e798..30f84a8f8 100644 --- a/charts/openshell-saw/templates/configmap-installer.yaml +++ b/charts/openshell-saw/templates/configmap-installer.yaml @@ -48,6 +48,31 @@ changes here reach an existing VM after a restart (unlike cloud-init). {{- if $webuiHost }}{{- $redirectUrl = printf "https://%s/oauth2/callback" $webuiHost }}{{- end }} {{- $dashboardEnabled := false }} {{- if and .Values.oidc.enabled .Values.dashboard.enabled .Values.route.webui }}{{- $dashboardEnabled = true }}{{- end }} +{{- /* +The existing golden-image verifier signs apply_bom.py in a fixed manifest. +Embed the canonical shared module INTO that signed file, instead of adding an +adjacent Python import that an older verifier would not cover. The deployed +installer and its runtime-user copy therefore remain one executable artifact. +*/ -}} +{{- $applyBom := .Files.Get "files/installer/apply_bom.py" -}} +{{- $sharedCore := required "canonical files/installer/saw_core.py is required" (.Files.Get "files/installer/saw_core.py") -}} +{{- $coreMarker := "_SAW_CORE_EMBEDDED_B64 = \"__SAW_CORE_SOURCE_BASE64__\"" -}} +{{- if ne (len (regexFindAll $coreMarker $applyBom -1)) 1 -}} +{{- fail "apply_bom.py must contain exactly one shared-core embedding marker" -}} +{{- end -}} +{{- $applyBom = replace $coreMarker (printf "_SAW_CORE_EMBEDDED_B64 = %q" (b64enc $sharedCore)) $applyBom -}} +{{- $adminSource := required "canonical files/installer/saw_admin.py is required" (.Files.Get "files/installer/saw_admin.py") -}} +{{- $adminMarker := "_SAW_ADMIN_EMBEDDED_B64 = \"__SAW_ADMIN_SOURCE_BASE64__\"" -}} +{{- if ne (len (regexFindAll $adminMarker $applyBom -1)) 1 -}} +{{- fail "apply_bom.py must contain exactly one admin-session embedding marker" -}} +{{- end -}} +{{- $applyBom = replace $adminMarker (printf "_SAW_ADMIN_EMBEDDED_B64 = %q" (b64enc $adminSource)) $applyBom -}} +{{- $uiSource := required "canonical files/installer/saw_ui_limit.py is required" (.Files.Get "files/installer/saw_ui_limit.py") -}} +{{- $uiMarker := "_SAW_UI_LIMIT_EMBEDDED_B64 = \"__SAW_UI_LIMIT_SOURCE_BASE64__\"" -}} +{{- if ne (len (regexFindAll $uiMarker $applyBom -1)) 1 -}} +{{- fail "apply_bom.py must contain exactly one UI transport embedding marker" -}} +{{- end -}} +{{- $applyBom = replace $uiMarker (printf "_SAW_UI_LIMIT_EMBEDDED_B64 = %q" (b64enc $uiSource)) $applyBom -}} {{- $signMode := .Values.signing.mode | default "warn" }} {{- if not (or (eq $signMode "off") (eq $signMode "warn") (eq $signMode "enforce")) }} {{- fail (printf "signing.mode must be off, warn, or enforce, got %q" $signMode) }} @@ -152,7 +177,7 @@ data: openshell-gateway-pki.conf: | {{ .Files.Get "files/guest/openshell-gateway-pki.conf" | indent 4 }} apply_bom.py: | -{{ .Files.Get "files/installer/apply_bom.py" | indent 4 }} +{{ $applyBom | indent 4 }} tool-actions.yaml: | {{ .Files.Get "files/installer/tool-actions.yaml" | indent 4 }} tool-gate.mjs: | diff --git a/charts/openshell-saw/templates/credential-proxy.yaml b/charts/openshell-saw/templates/credential-proxy.yaml index b44d4a709..b0d32e624 100644 --- a/charts/openshell-saw/templates/credential-proxy.yaml +++ b/charts/openshell-saw/templates/credential-proxy.yaml @@ -4,57 +4,13 @@ {{- fail "credentialProxy.image must be a repo@sha256: image; the chart default is the published multi-arch image, and an override (defaults.openshellSaw.credentialProxy.image, CREDENTIAL_PROXY_IMAGE) must also be digest-pinned." -}} {{- end -}} {{- $name := printf "%s-credential-proxy" (include "openshell-sandbox.fullname" .) -}} -{{- $profiles := dict -}} -{{- $refresh := dict -}} +{{- $catalog := dict -}} {{- range $path, $_ := .Files.Glob "files/provider-profiles/*.yaml" -}} - {{- $profile := $.Files.Get $path | fromYaml -}} - {{- range $credential := $profile.credentials | default list -}} - {{- $spec := $credential.refresh | default dict -}} - {{- if eq ($spec.strategy | default "") "oauth2_refresh_token" -}} - {{- $keys := list -}} - {{- range $spec.material -}}{{- $keys = append $keys .name -}}{{- end -}} - {{- range list "client_id" "client_secret" "refresh_token" -}} - {{- if not (has . $keys) -}}{{- fail "OAuth refresh requires client_id, client_secret and refresh_token material" -}}{{- end -}} - {{- end -}} - {{- $_ := set $refresh $profile.id (dict "credential" $credential.name "spec" (dict "strategy" $spec.strategy "tokenURL" $spec.token_url "refreshBeforeSeconds" $spec.refresh_before_seconds "maxLifetimeSeconds" $spec.max_lifetime_seconds "material" (dict "clientID" "client_id" "clientSecret" "client_secret" "refreshToken" "refresh_token"))) -}} - {{- end -}} - {{- end -}} - {{- if $profile.credentials -}} - {{- $hosts := list -}} - {{- range $profile.endpoints -}} - {{- if and .host (not (has .host $hosts)) -}}{{- $hosts = append $hosts .host -}}{{- end -}} - {{- end -}} - {{- $_ := set $profiles $profile.id $hosts -}} - {{- end -}} +{{- $_ := set $catalog $path ($.Files.Get $path | fromYaml) -}} {{- end -}} -{{- if hasKey $profiles "nvidia" -}}{{- $_ := set $profiles "build" (index $profiles "nvidia") -}}{{- end -}} -{{- $bindings := list -}} -{{- $mounts := dict -}} {{- $attached := concat (list .Values.inference.secretName) (.Values.additionalProviderSecrets | default list) | compact -}} -{{- range .Values.credentialProxy.secrets -}} - {{- if has .name $attached -}} - {{- include "openshell-sandbox.validateSecretName" .name -}} - {{- if not (regexMatch "^[A-Za-z0-9_.-]+$" (.key | default "")) -}} - {{- fail (printf "credentialProxy.secrets[%s].key is invalid" .name) -}} - {{- end -}} - {{- if not .providers -}}{{- fail (printf "credentialProxy.secrets[%s].providers is required" .name) -}}{{- end -}} - {{- if and (hasKey . "refresh") (not (kindIs "bool" .refresh)) -}} - {{- fail "credentialProxy.secrets[].refresh must be a boolean" -}} - {{- end -}} - {{- $binding := dict "secret" .name "key" .key "providers" .providers -}} - {{- if .refresh -}} - {{- if ne (len .providers) 1 -}}{{- fail "refresh binding requires exactly one provider type" -}}{{- end -}} - {{- $type := first .providers -}} - {{- if not (hasKey $refresh $type) -}}{{- fail (printf "provider %s has no OAuth refresh spec" $type) -}}{{- end -}} - {{- $entry := index $refresh $type -}} - {{- if ne .key $entry.credential -}}{{- fail "refresh binding key must match the profile credential" -}}{{- end -}} - {{- $_ := set $binding "refresh" $entry.spec -}} - {{- end -}} - {{- $bindings = append $bindings $binding -}} - {{- $_ := set $mounts .name true -}} - {{- end -}} -{{- end -}} -{{- $policy := dict "markerPrefix" "saw-cred" "allowHosts" (include "openshell-sandbox.egressHosts" . | fromJsonArray) "profiles" $profiles "bindings" $bindings -}} +{{- $policy := include "saw.governedProxyPolicy" (dict "catalog" $catalog "attached" $attached "bindings" .Values.credentialProxy.secrets "allowHosts" (include "openshell-sandbox.egressHosts" . | fromJsonArray)) | fromJson -}} +{{- $mounts := dict -}}{{- range $policy.bindings -}}{{- $_ := set $mounts .secret true -}}{{- end -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/openshell-saw/templates/deployment-mode.yaml b/charts/openshell-saw/templates/deployment-mode.yaml new file mode 100644 index 000000000..e1d7e576e --- /dev/null +++ b/charts/openshell-saw/templates/deployment-mode.yaml @@ -0,0 +1,2 @@ +{{/* Validate even when Helm schema validation is disabled. Emit no resource. */}} +{{- include "openshell-sandbox.validateDeploymentMode" . -}} diff --git a/charts/openshell-saw/values.schema.json b/charts/openshell-saw/values.schema.json new file mode 100644 index 000000000..c1a10172a --- /dev/null +++ b/charts/openshell-saw/values.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [ + "deployment" + ], + "properties": { + "deployment": { + "type": "object", + "additionalProperties": false, + "required": [ + "mode" + ], + "properties": { + "mode": { + "type": "string", + "enum": [ + "vm", + "container" + ], + "description": "Backend selection; only vm is currently implemented. Other modes fail rendering." + } + } + } + } +} diff --git a/charts/openshell-saw/values.yaml b/charts/openshell-saw/values.yaml index 75d155ffc..63aa6d87e 100644 --- a/charts/openshell-saw/values.yaml +++ b/charts/openshell-saw/values.yaml @@ -1,3 +1,9 @@ +# Deployment backend. VM remains the only implemented backend in this revision. +# container uses its separate composition chart; unsupported modes never +# silently fall back to a VM. See docs/container-workspaces.md. +deployment: + mode: vm + # Required — must be set at install time. Maximum 19 characters (OpenShell limit). sandboxName: "" # Operator SSH access (used by `make saw-configure` to fetch diff --git a/charts/saw-audit-writer/Chart.yaml b/charts/saw-audit-writer/Chart.yaml new file mode 100644 index 000000000..bc2bb746a --- /dev/null +++ b/charts/saw-audit-writer/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: saw-audit-writer +version: 0.1.0 +description: Restricted audit writer and workspace receipt forwarder diff --git a/charts/saw-audit-writer/files/forward.mjs b/charts/saw-audit-writer/files/forward.mjs new file mode 100644 index 000000000..3fc4a607c --- /dev/null +++ b/charts/saw-audit-writer/files/forward.mjs @@ -0,0 +1,46 @@ +import fs from 'node:fs'; +import https from 'node:https'; +import {randomUUID} from 'node:crypto'; +import {setTimeout as sleep} from 'node:timers/promises'; + +const dir='/exchange/log'; +fs.mkdirSync(dir,{recursive:true,mode:0o700}); +function request(id, body) { + return new Promise((resolve,reject) => { + const req=https.request(new URL('/v1/records/'+id,process.env.WRITER_URL), + {method:'POST',ca:fs.readFileSync('/trust/service-ca.crt'),timeout:3000,headers:{'content-length':body.length}}, res=>{ + let value=''; + res.setEncoding('utf8'); + res.on('data', chunk => {value+=chunk; if(value.length>128) res.destroy(new Error('oversize receipt'));}); + res.on('error',reject); + res.on('end',()=>res.statusCode===201 && /^[A-Za-z0-9+/]{86}==$/.test(value) + ? resolve(value) : reject(new Error('writer refused record'))); + }); + req.on('timeout',()=>req.destroy(new Error('writer timeout'))); + req.on('error',reject); req.end(body); + }); +} +async function forward(file) { + const id=file.slice(0,-4); + const fd=fs.openSync(dir+'/'+file,fs.constants.O_NOFOLLOW|fs.constants.O_NONBLOCK|fs.constants.O_RDONLY); + const bytes=Buffer.alloc(16385); + let size; + try { + if(!fs.fstatSync(fd).isFile()) return; + size=fs.readSync(fd,bytes,0,bytes.length,0); + if(size===bytes.length) return; + } finally {fs.closeSync(fd);} + const signature=await request(id,bytes.subarray(0,size)); + const tmp=dir+'/'+randomUUID()+'.tmp'; + fs.writeFileSync(tmp,signature,{flag:'wx',mode:0o600}); + fs.renameSync(tmp,dir+'/'+id+'.ack'); + fs.unlinkSync(dir+'/'+file); +} +for (;;) { + const files=await fs.promises.opendir(dir); + for await(const entry of files) { + if(!entry.isFile() || !/^[0-9a-f-]{36}\.req$/.test(entry.name)) continue; + try {await forward(entry.name);} catch { /* Retry published requests on the next scan. */ } + } + await sleep(100); +} diff --git a/charts/saw-audit-writer/files/writer.mjs b/charts/saw-audit-writer/files/writer.mjs new file mode 100644 index 000000000..2d108144a --- /dev/null +++ b/charts/saw-audit-writer/files/writer.mjs @@ -0,0 +1,56 @@ +import fs from 'node:fs'; +import https from 'node:https'; +import {createHash, createPrivateKey, createPublicKey, generateKeyPairSync, sign} from 'node:crypto'; +import {DatabaseSync} from 'node:sqlite'; + +const db = new DatabaseSync('/history/audit.db'); +db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; + CREATE TABLE IF NOT EXISTS metadata(key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS records(seq INTEGER PRIMARY KEY, id TEXT UNIQUE NOT NULL, body TEXT NOT NULL, + received TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);`); +const getKey = db.prepare("SELECT value FROM metadata WHERE key='signing-key'"); +if (!getKey.get()) { + const key = generateKeyPairSync('ed25519').privateKey.export({type:'pkcs8',format:'pem'}); + db.prepare("INSERT OR IGNORE INTO metadata VALUES ('signing-key',?)").run(key); +} +const signer = createPrivateKey(getKey.get().value); +const identity = JSON.parse(process.env.IDENTITY); +const pub = createPublicKey(signer).export({type:'spki',format:'der'}).toString('base64'); +console.log('receipt-public-key=' + pub); +const insert = db.prepare('INSERT OR IGNORE INTO records(id, body) VALUES (?, ?)'); +const lookup = db.prepare('SELECT body FROM records WHERE id=?'); +const tls = () => ({cert:fs.readFileSync('/tls/tls.crt'), key:fs.readFileSync('/tls/tls.key')}); +const server = https.createServer(tls(), async (req,res) => { + const reply = (status,text='') => {res.writeHead(status);res.end(text);}; + if (req.method === 'GET' && req.url === '/v1/receipt-key') return reply(200, pub); + const match = /^\/v1\/records\/([0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/.exec(req.url); + if (!match || req.method !== 'POST') return reply(404); + try { + const parts = []; let size=0; + for await (const part of req) { + size += part.length; + if (size > 16384) return reply(413); + parts.push(part); + } + const body = Buffer.concat(parts).toString('utf8'); + let record; + try {record=JSON.parse(body);} catch {return reply(400);} + if (record?.user?.subject !== identity.subject || record?.user?.username !== identity.username || + record.workspace !== identity.workspace || record.sandbox !== identity.sandbox || + !['allow','deny','approval_required','approved','rejected'].includes(record.decision) || + !/^sha256:[0-9a-f]{64}$/.test(record.argsDigest)) return reply(422); + const id = match[1]; + // Each insert is an autocommit SQLite transaction with synchronous=FULL. + insert.run(id,body); + if (lookup.get(id).body !== body) return reply(409); + const hash = createHash('sha256').update(body).digest('hex'); + reply(201,sign(null, Buffer.from('saw-audit-v1:'+id+':'+hash), signer).toString('base64')); + } catch { if (!res.headersSent) reply(503); } +}); +server.requestTimeout=10000; +server.listen(8443,'0.0.0.0'); + +// OpenShift rotates the projected certificate while the history claim stays mounted. +setInterval(() => { + try {server.setSecureContext(tls());} catch {console.error("cannot reload serving certificate");} +}, 60000).unref(); diff --git a/charts/saw-audit-writer/templates/writer.yaml b/charts/saw-audit-writer/templates/writer.yaml new file mode 100644 index 000000000..72d72e35e --- /dev/null +++ b/charts/saw-audit-writer/templates/writer.yaml @@ -0,0 +1,130 @@ +{{- if .Values.enabled }} +{{- $tn := required "trustedNamespace required" .Values.trustedNamespace }} +{{- $rn := required "runtimeNamespace required" .Values.runtimeNamespace }} +{{- if eq $tn $rn }}{{ fail "writer history requires a separate trusted namespace" }}{{ end }} +{{- $image := required "pinned Node 24 image required" .Values.image }} +{{- if not (regexMatch "@sha256:[0-9a-f]{64}$" $image) }}{{ fail "image must be digest pinned" }}{{ end }} +{{- $id := .Values.identity }} +{{- range $f := list "subject" "username" "workspace" "sandbox" }}{{- $_ := required $f (get $id $f) }}{{- end }} +{{- $base := .Release.Name }} +{{- range $name, $ns := dict "history" $tn "exchange" $rn "gate" $rn "workspace" $rn }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + namespace: {{ $ns }} + name: {{ if eq $name "exchange" }}saw-audit-{{ $base }}{{ else if eq $name "gate" }}saw-gate-{{ $base }}{{ else if eq $name "workspace" }}workspace-{{ $base }}{{ else }}{{ $base }}-{{ $name }}{{ end }} + annotations: {argocd.argoproj.io/sync-options: 'Prune=false,Delete=false'} + {{- if ne $name "history" }} + labels: + openshell.ai/sandbox-attachable: 'true' + openshell.ai/sandbox-attachable-workspace: {{ $id.workspace }} + saw.redhat.com/runtime-owner: {{ $.Values.ownerID }} + {{- end }} +spec: + storageClassName: {{ if or (eq $name "exchange") (and (eq $name "gate") $.Values.shareGateForModelSelection) }}{{ required "exchangeStorageClass required" $.Values.exchangeStorageClass }}{{ else }}{{ required "storageClass required" $.Values.storageClass }}{{ end }} + accessModes: [{{ if or (eq $name "exchange") (and (eq $name "gate") $.Values.shareGateForModelSelection) }}ReadWriteMany{{ else }}ReadWriteOnce{{ end }}] + resources: {requests: {storage: 1Gi}} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $base }} + namespace: {{ $tn }} + annotations: {service.beta.openshift.io/serving-cert-secret-name: {{ $base }}-tls} +spec: + selector: {saw-audit-writer: {{ $base }}} + ports: [{port: 8443, targetPort: 8443}] +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $base }}-ca + namespace: {{ $rn }} + annotations: {service.beta.openshift.io/inject-cabundle: 'true', argocd.argoproj.io/compare-options: IgnoreExtraneous} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $base }}, namespace: {{ $tn }}} +spec: + podSelector: {matchLabels: {saw-audit-writer: {{ $base }}}} + policyTypes: [Ingress,Egress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $rn }}}} + podSelector: {matchLabels: {saw-audit-forward: {{ $base }}}} + ports: [{port: 8443, protocol: TCP}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $base }}, namespace: {{ $rn }}} +spec: + podSelector: {matchLabels: {saw-audit-forward: {{ $base }}}} + policyTypes: [Ingress,Egress] + egress: + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $tn }}}} + podSelector: {matchLabels: {saw-audit-writer: {{ $base }}}} + ports: [{port: 8443, protocol: TCP}] + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: openshift-dns}} + podSelector: {matchLabels: {dns.operator.openshift.io/daemonset-dns: default}} + ports: [{port: 53, protocol: UDP}, {port: 53, protocol: TCP}, {port: 5353, protocol: UDP}, {port: 5353, protocol: TCP}] +{{- range $role, $ns := dict "writer" $tn "forward" $rn }} +{{- $script := $.Files.Get (printf "files/%s.mjs" $role) }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $base }}-{{ $role }}, namespace: {{ $ns }}} +data: + main.mjs: | +{{ $script | indent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: {{ $base }}-{{ $role }}, namespace: {{ $ns }}} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: {matchLabels: {saw-audit-{{ $role }}: {{ $base }}}} + template: + metadata: + labels: {saw-audit-{{ $role }}: {{ $base }}} + annotations: {saw-audit/source: {{ $script | sha256sum }}} + spec: + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: {{ $role }} + image: {{ $image }} + command: [node, /code/main.mjs] + resources: {requests: {cpu: 20m, memory: 64Mi}, limits: {cpu: 500m, memory: 256Mi}} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + env: + {{- if eq $role "writer" }} + - {name: IDENTITY, value: {{ $id | toJson | quote }}} + readinessProbe: {tcpSocket: {port: 8443}} + {{- else }} + - {name: WRITER_URL, value: {{ printf "https://%s.%s.svc:8443" $base $tn | quote }}} + {{- end }} + volumeMounts: + - {name: code, mountPath: /code, readOnly: true} + {{- if eq $role "writer" }} + - {name: data, mountPath: /history} + - {name: trust, mountPath: /tls, readOnly: true} + {{- else }} + - {name: data, mountPath: /exchange} + - {name: trust, mountPath: /trust, readOnly: true} + {{- end }} + volumes: + - {name: code, configMap: {name: {{ $base }}-{{ $role }}}} + {{- if eq $role "writer" }} + - {name: data, persistentVolumeClaim: {claimName: {{ $base }}-history}} + - {name: trust, secret: {secretName: {{ $base }}-tls}} + {{- else }} + - {name: data, persistentVolumeClaim: {claimName: saw-audit-{{ $base }}}} + - {name: trust, configMap: {name: {{ $base }}-ca}} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/saw-audit-writer/values.yaml b/charts/saw-audit-writer/values.yaml new file mode 100644 index 000000000..57ae10487 --- /dev/null +++ b/charts/saw-audit-writer/values.yaml @@ -0,0 +1,8 @@ +enabled: false +trustedNamespace: "" +runtimeNamespace: "" +image: "" +storageClass: "" +ownerID: "" +identity: {} +exchangeStorageClass: "" diff --git a/charts/saw-bom/profiles/ci-nvidia/default/sandbox.yaml b/charts/saw-bom/profiles/ci-nvidia/default/sandbox.yaml index 3a587064a..7ec2c9295 100644 --- a/charts/saw-bom/profiles/ci-nvidia/default/sandbox.yaml +++ b/charts/saw-bom/profiles/ci-nvidia/default/sandbox.yaml @@ -30,7 +30,7 @@ spec: type: nemoclaw enabled: false agent: openclaw - image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox:latest + image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd providers: - nvidia - name: toolbox diff --git a/charts/saw-bom/profiles/data-science/cuda-dev/sandbox.yaml b/charts/saw-bom/profiles/data-science/cuda-dev/sandbox.yaml index a47d81555..b8ad71d66 100644 --- a/charts/saw-bom/profiles/data-science/cuda-dev/sandbox.yaml +++ b/charts/saw-bom/profiles/data-science/cuda-dev/sandbox.yaml @@ -7,7 +7,7 @@ spec: type: nemoclaw enabled: true agent: openclaw - image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox:latest + image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd providers: - nvidia # How the agent reaches its model: the `nvidia` provider's credential diff --git a/charts/saw-bom/profiles/data-science/default/sandbox.yaml b/charts/saw-bom/profiles/data-science/default/sandbox.yaml index 41c7c2e8c..3b00be35b 100644 --- a/charts/saw-bom/profiles/data-science/default/sandbox.yaml +++ b/charts/saw-bom/profiles/data-science/default/sandbox.yaml @@ -60,7 +60,7 @@ spec: type: nemoclaw enabled: false agent: openclaw - image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox:latest + image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd providers: - nvidia - name: toolbox diff --git a/charts/saw-bom/profiles/personal-assistant/personal-assistant/sandbox.yaml b/charts/saw-bom/profiles/personal-assistant/personal-assistant/sandbox.yaml index 83e828a82..a42da1757 100644 --- a/charts/saw-bom/profiles/personal-assistant/personal-assistant/sandbox.yaml +++ b/charts/saw-bom/profiles/personal-assistant/personal-assistant/sandbox.yaml @@ -9,7 +9,7 @@ spec: agent: openclaw # NemoClaw v0.0.133 (#101): OpenClaw 2026.9.2, which loads the bundle's # MCP servers (2026.7.1 loaded only its skills). - image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox:latest + image: ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd providers: - nvidia - slack diff --git a/charts/saw-image-cache/Chart.yaml b/charts/saw-image-cache/Chart.yaml new file mode 100644 index 000000000..5e3c94bb3 --- /dev/null +++ b/charts/saw-image-cache/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: saw-image-cache +description: Prepare approved workspace images on eligible nodes before sandbox creation +type: application +version: 0.1.0 diff --git a/charts/saw-image-cache/templates/cache.yaml b/charts/saw-image-cache/templates/cache.yaml new file mode 100644 index 000000000..c0e8c6ca5 --- /dev/null +++ b/charts/saw-image-cache/templates/cache.yaml @@ -0,0 +1,74 @@ +{{- if .Values.enabled -}} +{{- if not .Values.images -}}{{ fail "saw-image-cache requires a nonempty list of site-approved image digests" }}{{- end -}} +{{- $images := uniq .Values.images -}} +{{- range $image := append $images .Values.holderImage -}} +{{- if not (regexMatch "^[^[:space:]@]+@sha256:[0-9a-f]{64}$" $image) -}} +{{- fail "saw-image-cache images must be pinned by sha256 digest" -}} +{{- end -}} +{{- end }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }} +automountServiceAccountToken: false +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ .Release.Name }} +spec: + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + serviceAccountName: {{ .Release.Name }} + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + seccompProfile: {type: RuntimeDefault} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{ . | toJson }} + {{- end }} + nodeSelector: {{ .Values.nodeSelector | toJson }} + {{- with .Values.tolerations }} + tolerations: {{ . | toJson }} + {{- end }} + containers: + - name: holder + image: {{ .Values.holderImage | quote }} + command: [/usr/bin/sleep, infinity] + resources: + requests: {cpu: 5m, memory: 12Mi} + limits: {cpu: 50m, memory: 32Mi} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + volumeMounts: + {{- range $i, $_ := $images }} + - name: image-{{ $i }} + mountPath: /images/{{ $i }} + readOnly: true + {{- end }} + volumes: + {{- range $i, $image := $images }} + - name: image-{{ $i }} + image: + reference: {{ $image | quote }} + pullPolicy: IfNotPresent + {{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + policyTypes: [Ingress, Egress] +{{- end }} diff --git a/charts/saw-image-cache/values.yaml b/charts/saw-image-cache/values.yaml new file mode 100644 index 000000000..5d51c169c --- /dev/null +++ b/charts/saw-image-cache/values.yaml @@ -0,0 +1,9 @@ +enabled: false +# Site-approved OCI images used by the workload. Every reference must be immutable. +images: [] +# This infrastructure image runs only sleep. Workspace images are mounted read-only. +holderImage: registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c +nodeSelector: + kubernetes.io/os: linux +tolerations: [] +imagePullSecrets: [] diff --git a/charts/saw-kubernetes-controller/Chart.yaml b/charts/saw-kubernetes-controller/Chart.yaml new file mode 100644 index 000000000..00c4305e1 --- /dev/null +++ b/charts/saw-kubernetes-controller/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: saw-kubernetes-controller +version: 0.1.2 +type: application diff --git a/charts/saw-kubernetes-controller/templates/controller.yaml b/charts/saw-kubernetes-controller/templates/controller.yaml new file mode 100644 index 000000000..029a0b1ab --- /dev/null +++ b/charts/saw-kubernetes-controller/templates/controller.yaml @@ -0,0 +1,139 @@ +{{- if .Values.enabled }} +{{- $c := .Values.configuration -}} +{{- $t := .Release.Namespace -}} +{{- $n := $c.namespace -}} +{{- $g := $c.gateway -}} +{{- $gov := $c.governanceNamespace -}} +{{- $name := printf "%s-reconciler" $g -}} +{{- $policy := printf "saw-%s-runtime" $n -}} +{{- if or (eq $t $n) (ne $c.trustedNamespace $t) (not .Values.claimNames) (not .Values.trustedEgress) (not $c.approved) (not .Values.stateClaim) }}{{ fail "dedicated namespaces, approved boundary, retained claims and explicit egress required" }}{{ end -}} +{{- if not (regexMatch "^.+@sha256:[0-9a-f]{64}$" .Values.image) }}{{ fail "controller image digest required" }}{{ end -}} +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $name }}} +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $name }}} +data: + controller.json: {{ $c | toJson | quote }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $c.status }} + namespace: {{ $n }} + labels: {saw.redhat.com/runtime-owner: {{ $c.ownerID }}, saw.redhat.com/managed-by: kubernetes-runtime-controller} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: {name: {{ $t }}-{{ $name }}} +rules: + - {apiGroups: [""], resources: [namespaces], resourceNames: [{{ $n }}], verbs: [get]} + - {apiGroups: [""], resources: [nodes], verbs: [list]} + {{- if eq $c.mode "kata" }} + - {apiGroups: [node.k8s.io], resources: [runtimeclasses], resourceNames: [{{ required "approved RuntimeClass name required" $c.workloadRuntimeClass | quote }}], verbs: [get]} + - {apiGroups: [admissionregistration.k8s.io], resources: [validatingadmissionpolicies, validatingadmissionpolicybindings], resourceNames: [{{ $policy }}-kata], verbs: [get]} + {{- end }} + - {apiGroups: [policy.networking.k8s.io], resources: [adminnetworkpolicies], verbs: [list]} + - {apiGroups: [policy.networking.k8s.io], resources: [adminnetworkpolicies], resourceNames: [{{ $policy }}-egress], verbs: [get]} + - {apiGroups: [admissionregistration.k8s.io], resources: [validatingadmissionpolicies, validatingadmissionpolicybindings], resourceNames: [{{ $policy }}, {{ $n }}-pods-driver, {{ $n }}-secrets-driver, {{ $n }}-services-driver], verbs: [get]} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: {name: {{ $t }}-{{ $name }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ $t }}-{{ $name }}} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ $t }}}] +{{- range $ns := list $n $t $gov }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $name }}, namespace: {{ $ns }}} +rules: +{{- if eq $ns $n }} + - {apiGroups: [rbac.authorization.k8s.io], resources: [roles], resourceNames: [{{ $g }}-gateway], verbs: [get]} + # Public trust only; no private administrator identity is stored in this Secret. + - {apiGroups: [""], resources: [secrets], resourceNames: [{{ $g }}-client-ca], verbs: [get, update]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [{{ $c.profile }}, {{ $c.intent }}, {{ $g }}-gateway-config, {{ $g }}-proxy-ca], verbs: [get]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [{{ $c.intent }}], verbs: [watch]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [{{ $c.status }}], verbs: [get, update]} + - {apiGroups: [""], resources: [persistentvolumeclaims], resourceNames: {{ $.Values.claimNames | toJson }}, verbs: [get]} + - {apiGroups: [apps], resources: [statefulsets], resourceNames: [{{ $g }}-gateway], verbs: [get]} + - {apiGroups: [""], resources: [services], resourceNames: [{{ $g }}-gateway], verbs: [get]} + - {apiGroups: [networking.k8s.io], resources: [networkpolicies], resourceNames: [{{ $g }}-gateway, {{ $g }}-default-deny, openshell-sandbox-workloads, openshell-sandbox-supervisors], verbs: [get]} +{{- else if eq $ns $t }} + - {apiGroups: [""], resources: [pods], verbs: [get]} + - {apiGroups: [apps], resources: [deployments], resourceNames: [{{ $g }}-credential-proxy], verbs: [get]} + - {apiGroups: [networking.k8s.io], resources: [networkpolicies], resourceNames: [{{ $g }}-credential-proxy], verbs: [get]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [{{ $c.proxyPolicy }}, {{ $g }}-gateway-ca], verbs: [get]} +{{- else }} + - {apiGroups: [apps], resources: [deployments], resourceNames: [governance-interceptor], verbs: [get]} + - {apiGroups: [""], resources: [services], resourceNames: [governance-interceptor], verbs: [get]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [governance-interceptor-policy, governance-interceptor-profiles], verbs: [get]} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $name }}, namespace: {{ $ns }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }}} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ $t }}}] +{{- end }} +{{- if $.Values.imageBuildNamespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $name }}-image-pull, namespace: {{ $.Values.imageBuildNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: system:image-puller} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ $t }}}] +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $name }}} +spec: + podSelector: {matchLabels: {app.kubernetes.io/instance: {{ $g }}, openshell.pattern/role: reconciler}} + policyTypes: [Ingress,Egress] + ingress: [] + egress: +{{ toYaml $.Values.trustedEgress | indent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: {{ $name }}} +spec: + replicas: 1 + strategy: {type: Recreate} + selector: {matchLabels: {app.kubernetes.io/instance: {{ $g }}, openshell.pattern/role: reconciler}} + template: + metadata: + labels: {app.kubernetes.io/instance: {{ $g }}, openshell.pattern/role: reconciler} + annotations: {checksum/config: {{ $c | toJson | sha256sum | quote }}} + spec: + serviceAccountName: {{ $name }} + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: reconciler + image: {{ $.Values.image | quote }} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + resources: {requests: {cpu: 50m, memory: 96Mi}, limits: {memory: 256Mi}} + env: + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_UID, valueFrom: {fieldRef: {fieldPath: metadata.uid}}} + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: api, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true} + - {name: state, mountPath: /state} + - {name: admin-session, mountPath: /run/saw-admin} + volumes: + - {name: config, configMap: {name: {{ $name }}}} + - {name: admin-session, emptyDir: {medium: Memory, sizeLimit: 4Mi}} + - {name: state, persistentVolumeClaim: {claimName: {{ $.Values.stateClaim }}}} + - name: api + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: {path: token, expirationSeconds: 600} + - configMap: {name: kube-root-ca.crt, items: [{key: ca.crt, path: ca.crt}]} +{{- end }} diff --git a/charts/saw-kubernetes-controller/values.yaml b/charts/saw-kubernetes-controller/values.yaml new file mode 100644 index 000000000..336a0610b --- /dev/null +++ b/charts/saw-kubernetes-controller/values.yaml @@ -0,0 +1,7 @@ +enabled: false +image: "" +configuration: {} +stateClaim: "" +claimNames: [] +trustedEgress: [] +imageBuildNamespace: "" diff --git a/charts/saw-kubernetes-identity/Chart.yaml b/charts/saw-kubernetes-identity/Chart.yaml new file mode 100644 index 000000000..dedf208fe --- /dev/null +++ b/charts/saw-kubernetes-identity/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: saw-kubernetes-identity +description: Create-once identity bootstrap for a stock OpenShell Kubernetes gateway +type: application +version: 0.1.0 diff --git a/charts/saw-kubernetes-identity/files/bootstrap.py b/charts/saw-kubernetes-identity/files/bootstrap.py new file mode 100644 index 000000000..0254c1bda --- /dev/null +++ b/charts/saw-kubernetes-identity/files/bootstrap.py @@ -0,0 +1,292 @@ +"""Create-once gateway identity; only selected leaf keys leave the trusted namespace. + +The stock gateway generates PKI into a memory-backed volume. This helper retains +one atomic bundle, then resumes its fixed projections. It never adopts or +replaces identity material. Administrator clients use per-operation sessions; +no administrator private key is retained. +""" +import base64 +import hashlib +import json +import os +from pathlib import Path +import secrets +import ssl +import subprocess +import sys +from urllib.error import HTTPError, URLError +from urllib.request import Request, HTTPSHandler, HTTPRedirectHandler, build_opener +import uuid +import re + +OWNER = "saw.redhat.com/runtime-owner" +HASH = "saw.redhat.com/identity-config-sha256" +KEEP = "Prune=false,Delete=false" +LIMIT = 1024 * 1024 +NAME = re.compile(r"[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\Z") + + +class Failure(Exception): + pass + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def encode(raw): + return base64.b64encode(raw).decode("ascii") + + +def decode(raw): + try: + value = base64.b64decode(raw, validate=True) + if not value or len(value) > LIMIT: + raise ValueError() + return value + except (ValueError, TypeError): + raise Failure("invalid retained identity encoding") from None + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +class Api: + def __init__(self): + root = Path("/var/run/secrets/kubernetes.io/serviceaccount") + self.token = root / "token" + self.opener = build_opener(NoRedirect(), HTTPSHandler( + context=ssl.create_default_context(cafile=str(root / "ca.crt")))) + + def request(self, method, path, body=None): + token = self.token.read_text().strip() + if not token or any(c.isspace() for c in token): + raise Failure("invalid bootstrap service account token") + request = Request("https://kubernetes.default.svc" + path, + data=canonical(body).encode() if body is not None else None, + headers={"Authorization": "Bearer " + token, + "Content-Type": "application/json"}, method=method) + try: + with self.opener.open(request, timeout=20) as response: + raw = response.read(LIMIT + 1) + if len(raw) > LIMIT: + raise Failure("oversized API response") + value = json.loads(raw) + if not isinstance(value, dict): + raise Failure("invalid API response") + return value + except HTTPError as exc: + code = exc.code + exc.close() + if method == "GET" and code == 404: + return None + if method == "POST" and code == 409: + return None # caller observes the winner, never overwrites it + raise Failure(f"bootstrap API failed ({code})") from None + except (URLError, OSError, ValueError): + raise Failure("bootstrap API unavailable") from None + + def namespace(self, name): + return self.request("GET", f"/api/v1/namespaces/{name}") + + def get(self, namespace, name): + return self.request("GET", f"/api/v1/namespaces/{namespace}/secrets/{name}") + + def create(self, namespace, secret): + self.request("POST", f"/api/v1/namespaces/{namespace}/secrets", secret) + + +def validate(config): + if set(config) != {"ownerID", "namespace", "targetNamespace", "gateway", "serverSans"}: + raise Failure("invalid bootstrap configuration fields") + for key in ("namespace", "targetNamespace", "gateway"): + value = config[key] + if (not isinstance(value, str) or not NAME.fullmatch(value) or len(value) > 50 + or value == "default" or value.startswith(("openshift", "kube-"))): + raise Failure("invalid bootstrap resource name") + if config["namespace"] == config["targetNamespace"]: + raise Failure("gateway identity requires a separate trusted namespace") + try: + owner = uuid.UUID(config["ownerID"]) + if str(owner) != config["ownerID"] or owner.int == 0: + raise ValueError() + except (ValueError, TypeError, AttributeError): + raise Failure("invalid installation identity") from None + names = config["serverSans"] + if (not isinstance(names, list) or not names or len(names) > 12 + or any(not isinstance(n, str) or len(n) > 253 + or not all(NAME.fullmatch(p) for p in n.split(".")) for n in names) + or len(set(names)) != len(names)): + raise Failure("server SANs must be unique DNS names") + expected = config["gateway"] + "-gateway." + config["targetNamespace"] + ".svc" + if expected not in names or expected + ".cluster.local" not in names: + raise Failure("gateway Service DNS SANs are required") + + +def openssl(*args): + try: + result = subprocess.run(["openssl", *map(str, args)], capture_output=True, + timeout=20, check=False) + if result.returncode: + raise Failure("identity cryptographic validation failed") + return result.stdout + except (OSError, subprocess.SubprocessError): + raise Failure("identity cryptographic tool unavailable") from None + + +def generate(config, directory): + """Consume stock generate-certs output; CA key never enters a Secret.""" + directory = Path(directory) + sources = {"ca": "ca.crt", "serverCert": "server/tls.crt", "serverKey": "server/tls.key", + "signing": "jwt/signing.pem", "public": "jwt/public.pem", "kid": "jwt/kid"} + data = {} + for key, filename in sources.items(): + path = directory / filename + if path.is_symlink() or not path.is_file(): + raise Failure("stock gateway generated incomplete identity") + data[key] = encode(path.read_bytes()) + ca = directory / "ca.crt" + cert = directory / "server/tls.crt" + for name in config["serverSans"]: + openssl("verify", "-CAfile", ca, "-verify_hostname", name, cert) + data["kek"] = encode(base64.b64encode(secrets.token_bytes(32))) + (directory / "ca.key").unlink(missing_ok=True) + return data + + +def projections(config, data): + required = {"ca", "serverCert", "serverKey", "signing", "public", "kid", "kek"} + if set(data) != required: + raise Failure("retained identity bundle is incomplete") + for value in data.values(): + decode(value) + if len(decode(decode(data["kek"]))) != 32: + raise Failure("invalid retained credential encryption key") + ns, target, gateway = config["namespace"], config["targetNamespace"], config["gateway"] + return [ + (target, gateway + "-server", "kubernetes.io/tls", + {"ca.crt": data["ca"], "tls.crt": data["serverCert"], "tls.key": data["serverKey"]}), + (target, gateway + "-client-ca", "Opaque", {"base.crt": data["ca"], "ca.crt": encode(decode(data["ca"]).rstrip()+b"\n")}), + (target, gateway + "-jwt", "Opaque", + {"signing.pem": data["signing"], "public.pem": data["public"], "kid": data["kid"]}), + (target, gateway + "-kek", "Opaque", {"key-encryption-key": data["kek"]}), + ] + + +def secret(config, revision, namespace, name, kind, data): + return {"apiVersion": "v1", "kind": "Secret", "type": kind, "immutable": not name.endswith("-client-ca"), + "metadata": {"name": name, "namespace": namespace, "labels": {OWNER: config["ownerID"]}, + "annotations": {HASH: revision, "argocd.argoproj.io/sync-options": KEEP}}, + "data": data} + + +def check(actual, expected): + if not actual: + raise Failure("identity Secret missing after create") + actual = json.loads(json.dumps(actual)) + if expected["immutable"] is False: + data = actual.get("data", {}) + if data.get("base.crt") != expected["data"]["base.crt"] or not data.get("ca.crt"): + raise Failure("client trust baseline changed") + # Only public certificates are allowed in the mutable session bundle. + for key, value in data.items(): + if key not in {"base.crt", "ca.crt"} and not re.fullmatch(r"session-[a-f0-9-]{36}\.crt", key): + raise Failure("unexpected client trust material") + pem = decode(value) + if b"PRIVATE KEY" in pem: + raise Failure("private material in client trust") + actual["data"] = expected["data"] + meta, wanted = actual.get("metadata", {}), expected["metadata"] + if (meta.get("deletionTimestamp") or meta.get("ownerReferences") + or any(meta.get(k) != wanted[k] for k in ("name", "namespace")) + or meta.get("labels", {}).get(OWNER) != wanted["labels"][OWNER] + or any(meta.get("annotations", {}).get(k) != v for k, v in wanted["annotations"].items()) + or actual.get("immutable") is not expected["immutable"] or actual.get("type") != expected["type"] + or actual.get("data") != expected["data"]): + raise Failure("existing identity is foreign, changed or deleting; no automatic adoption or rotation") + + +def run(api, config, generator): + validate(config) + # Argo can cache Helm capabilities across a CRD install. Check the actual + # APIs before publishing any gateway credential, including on a replay. + for group, resource, kind, namespaced in ( + ("agents.x-k8s.io/v1beta1", "sandboxes", "Sandbox", True), + ("policy.networking.k8s.io/v1alpha1", "adminnetworkpolicies", "AdminNetworkPolicy", False), + ): + listing = api.request("GET", "/apis/" + group) or {} + if listing.get("groupVersion") != group or not any( + entry.get("name") == resource and entry.get("kind") == kind + and entry.get("namespaced") is namespaced for entry in listing.get("resources", []) + ): + raise Failure(f"required Kubernetes API is unavailable: {group}/{kind}") + identities = {} + for ns in (config["namespace"], config["targetNamespace"]): + meta = (api.namespace(ns) or {}).get("metadata", {}) + if (not meta.get("uid") or meta.get("deletionTimestamp") + or meta.get("labels", {}).get(OWNER) != config["ownerID"]): + raise Failure("bootstrap namespace missing, deleting or owned by another installation") + identities[ns] = meta["uid"] + revision = hashlib.sha256(canonical({"config": config, "namespaces": identities}).encode()).hexdigest() + ns, gateway = config["namespace"], config["gateway"] + public_path = f"/api/v1/namespaces/{ns}/configmaps/{gateway}-gateway-ca" + retained = api.get(ns, gateway + "-identity") + if retained is None: + names = [(config["targetNamespace"], gateway + suffix) + for suffix in ("-server", "-client-ca", "-jwt", "-kek")] + if (any(api.get(n, name) is not None for n, name in names) + or api.request("GET", public_path) is not None): + raise Failure("orphaned identity projections; restore retained bundle, do not regenerate") + data = generator() + projections(config, data) # reject invalid material before first side effect + api.create(ns, secret(config, revision, ns, gateway + "-identity", "Opaque", data)) + retained = api.get(ns, gateway + "-identity") + # A concurrent creator wins atomically. All projections use its bundle. + data = (retained or {}).get("data", {}) + check(retained, secret(config, revision, ns, gateway + "-identity", "Opaque", data)) + projected = projections(config, data) + public = {"apiVersion": "v1", "kind": "ConfigMap", "immutable": True, + "metadata": {"name": gateway+"-gateway-ca", "namespace": ns, + "labels": {OWNER: config["ownerID"]}, + "annotations": {HASH: revision}}, + "data": {"ca.crt": decode(data["ca"]).decode("ascii")}} + path = f"/api/v1/namespaces/{ns}/configmaps" + existing = api.request("GET", path+"/"+public["metadata"]["name"]) + if existing is None: + api.request("POST", path, public) + existing = api.request("GET", path+"/"+public["metadata"]["name"]) + if (not existing or existing.get("immutable") is not True or existing.get("data") != public["data"] + or existing.get("metadata", {}).get("labels", {}).get(OWNER) != config["ownerID"] + or existing.get("metadata", {}).get("annotations", {}).get(HASH) != revision + or existing.get("metadata", {}).get("ownerReferences") + or existing.get("metadata", {}).get("deletionTimestamp")): + raise Failure("public gateway CA projection is foreign or changed") + for namespace, name, kind, values in projected: + expected = secret(config, revision, namespace, name, kind, values) + existing = api.get(namespace, name) + if existing is None: + api.create(namespace, expected) + existing = api.get(namespace, name) + check(existing, expected) + + +def main(): + os.umask(0o077) + try: + config = json.loads(Path("/config/config.json").read_text()) + run(Api(), config, lambda: generate(config, "/generated/pki")) + print("Gateway identity retained and projections verified; no keys rotated.") + return 0 + except Failure as exc: + print("Identity bootstrap failed: " + str(exc), file=sys.stderr) + return 1 + except (OSError, ValueError): + # Never emit API bodies, subprocess stderr, PEMs or credential-bearing data. + print("Identity bootstrap failed; inspect configuration, ownership and retained state.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/charts/saw-kubernetes-identity/templates/bootstrap.yaml b/charts/saw-kubernetes-identity/templates/bootstrap.yaml new file mode 100644 index 000000000..f0edf2925 --- /dev/null +++ b/charts/saw-kubernetes-identity/templates/bootstrap.yaml @@ -0,0 +1,174 @@ +{{- if .Values.enabled }} +{{- $v := .Values -}} +{{- range $name := list .Release.Namespace $v.targetNamespace $v.gateway }} +{{- if or (gt (len $name) 50) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $name)) (eq $name "default") (hasPrefix "openshift" $name) (hasPrefix "kube-" $name) }} +{{- fail "bootstrap requires explicit dedicated DNS-label names, at most 50 characters" }} +{{- end }}{{- end }} +{{- if eq .Release.Namespace $v.targetNamespace }}{{- fail "trusted namespace must differ from runtime namespace" }}{{- end }} +{{- if not (regexMatch "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" $v.ownerID) }}{{- fail "explicit installation UUID required" }}{{- end }} +{{- range $image := list $v.gatewayImage $v.helperImage }} +{{- if not (regexMatch "^.+@sha256:[0-9a-f]{64}$" $image) }}{{- fail "bootstrap images must be digest pinned" }}{{- end }}{{- end }} +{{- $svc := printf "%s-gateway.%s.svc" $v.gateway $v.targetNamespace }} +{{- if or (not (has $svc $v.serverSans)) (not (has (printf "%s.cluster.local" $svc) $v.serverSans)) }}{{- fail "both gateway Service DNS SANs are required" }}{{- end }} +{{- $config := dict "ownerID" $v.ownerID "namespace" .Release.Namespace "targetNamespace" $v.targetNamespace "gateway" $v.gateway "serverSans" $v.serverSans }} +{{- $name := printf "%s-identity-bootstrap" $v.gateway }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }} + annotations: + argocd.argoproj.io/sync-wave: "-20" +data: + config.json: {{ $config | toJson | quote }} + bootstrap.py: | +{{ .Files.Get "files/bootstrap.py" | indent 4 }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + annotations: + argocd.argoproj.io/sync-wave: "-20" +automountServiceAccountToken: false +{{- range $namespace := list .Release.Namespace $v.targetNamespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ $namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-20" +rules: + # Kubernetes RBAC cannot restrict create by resourceNames. Both namespaces + # are dedicated, admin-owned; this short-lived SA is part of the trusted base. + - apiGroups: [""] + resources: [secrets] + verbs: [create] + - apiGroups: [""] + resources: [secrets] + verbs: [get] + resourceNames: +{{- if eq $namespace $.Release.Namespace }} + - {{ printf "%s-identity" $v.gateway }} +{{- else }} + - {{ printf "%s-server" $v.gateway }} + - {{ printf "%s-client-ca" $v.gateway }} + - {{ printf "%s-jwt" $v.gateway }} + - {{ printf "%s-kek" $v.gateway }} +{{- end }} +{{- if eq $namespace $.Release.Namespace }} + - {apiGroups: [""], resources: [configmaps], verbs: [create]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [{{ printf "%s-gateway-ca" $v.gateway }}], verbs: [get]} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ $namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-20" +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }}} +subjects: + - {kind: ServiceAccount, name: {{ $name }}, namespace: {{ $.Release.Namespace }}} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + argocd.argoproj.io/sync-wave: "-20" +rules: + - apiGroups: [""] + resources: [namespaces] + resourceNames: [{{ .Release.Namespace }}, {{ $v.targetNamespace }}] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + argocd.argoproj.io/sync-wave: "-20" +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ .Release.Namespace }}-{{ $name }}} +subjects: + - {kind: ServiceAccount, name: {{ $name }}, namespace: {{ .Release.Namespace }}} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded + argocd.argoproj.io/sync-wave: "-10" +spec: + backoffLimit: 2 + activeDeadlineSeconds: 300 + template: + metadata: + labels: + openshell.pattern/role: identity-bootstrap + spec: + restartPolicy: Never + serviceAccountName: {{ $name }} + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + seccompProfile: {type: RuntimeDefault} + initContainers: + - name: stock-certificates + image: {{ $v.gatewayImage }} + args: + - generate-certs + - --output-dir + - /generated/pki +{{- range $v.serverSans }} + - --server-san + - {{ . | quote }} +{{- end }} + env: + - {name: HOME, value: /generated/home} + - {name: XDG_CONFIG_HOME, value: /generated/config} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 256Mi} + volumeMounts: + - {name: generated, mountPath: /generated} + containers: + - name: retain-and-project + image: {{ $v.helperImage }} + command: [python3, /config/bootstrap.py] + env: + - {name: PYTHONDONTWRITEBYTECODE, value: "1"} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {memory: 256Mi} + volumeMounts: + - {name: generated, mountPath: /generated} + - {name: config, mountPath: /config, readOnly: true} + - {name: api, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true} + volumes: + - name: generated + emptyDir: {medium: Memory, sizeLimit: 16Mi} + - name: config + configMap: {name: {{ $name }}} + - name: api + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: {path: token, expirationSeconds: 600} + - configMap: + name: kube-root-ca.crt + items: [{key: ca.crt, path: ca.crt}] +{{- end }} diff --git a/charts/saw-kubernetes-identity/values.schema.json b/charts/saw-kubernetes-identity/values.schema.json new file mode 100644 index 000000000..4454aa255 --- /dev/null +++ b/charts/saw-kubernetes-identity/values.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "global": { + "type": "object" + }, + "enabled": { + "type": "boolean" + }, + "ownerID": { + "type": "string" + }, + "targetNamespace": { + "type": "string" + }, + "gateway": { + "type": "string" + }, + "serverSans": { + "type": "array", + "maxItems": 12, + "uniqueItems": true, + "items": { + "type": "string", + "maxLength": 253, + "pattern": "^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" + } + }, + "gatewayImage": { + "type": "string" + }, + "helperImage": { + "type": "string" + } + } +} diff --git a/charts/saw-kubernetes-identity/values.yaml b/charts/saw-kubernetes-identity/values.yaml new file mode 100644 index 000000000..5d9822b0c --- /dev/null +++ b/charts/saw-kubernetes-identity/values.yaml @@ -0,0 +1,8 @@ +enabled: false +ownerID: "" +# Install this chart in a dedicated trusted namespace; target is separate. +targetNamespace: "" +gateway: "" +serverSans: [] +gatewayImage: quay.io/opendatahub/odh-openshell-gateway@sha256:bd2d7c9084b80b8d3d8ef92d00a27b963b35168fad84cd862b7f91fd9c488c3e +helperImage: registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c diff --git a/charts/saw-kubernetes-proxy/Chart.yaml b/charts/saw-kubernetes-proxy/Chart.yaml new file mode 100644 index 000000000..cee97f759 --- /dev/null +++ b/charts/saw-kubernetes-proxy/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: saw-kubernetes-proxy +description: Existing SAW credential proxy in a separate trusted namespace +type: application +version: 0.1.0 diff --git a/charts/saw-kubernetes-proxy/files/project_ca.py b/charts/saw-kubernetes-proxy/files/project_ca.py new file mode 100644 index 000000000..814e982b0 --- /dev/null +++ b/charts/saw-kubernetes-proxy/files/project_ca.py @@ -0,0 +1,145 @@ +"""Project only the stock proxy's public CA; refuse replacement or adoption. + +The preceding init container runs the unchanged SAW proxy init-ca command. +This helper never reads a Secret, returns credential data or rotates trust. +""" +import json +from pathlib import Path +import re +import ssl +import sys +from urllib.error import HTTPError, URLError +from urllib.request import Request, HTTPSHandler, HTTPRedirectHandler, build_opener +import uuid + +OWNER = "saw.redhat.com/runtime-owner" +KEEP = "Prune=false,Delete=false" +NAME = re.compile(r"[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\Z") +LIMIT = 128 * 1024 + + +class Failure(Exception): + pass + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +class Api: + def __init__(self): + root = Path('/var/run/secrets/kubernetes.io/serviceaccount') + self.token = root / 'token' + self.opener = build_opener(NoRedirect(), HTTPSHandler( + context=ssl.create_default_context(cafile=str(root / 'ca.crt')))) + + def call(self, method, path, body=None): + try: + token = self.token.read_text().strip() + if not token or any(c.isspace() for c in token): + raise ValueError() + data = None if body is None else json.dumps(body).encode() + request = Request('https://kubernetes.default.svc' + path, data=data, + headers={'Authorization': 'Bearer ' + token, + 'Content-Type': 'application/json'}, method=method) + with self.opener.open(request, timeout=20) as response: + raw = response.read(LIMIT + 1) + if len(raw) > LIMIT: + raise ValueError() + result = json.loads(raw) + if not isinstance(result, dict): + raise ValueError() + return result + except HTTPError as exc: + status = exc.code + exc.close() + if (method == 'GET' and status == 404) or (method == 'POST' and status == 409): + return None + raise Failure(f'projection API failed ({status})') from None + except (OSError, ValueError, URLError): + raise Failure('projection API unavailable') from None + + +def validate(config): + if not isinstance(config, dict) or set(config) != {'ownerID', 'namespace', 'targetNamespace', 'name'}: + raise Failure('invalid projection fields') + for key in ('namespace', 'targetNamespace', 'name'): + name = config[key] + if (not isinstance(name, str) or len(name) > 63 or not NAME.fullmatch(name) + or name == 'default' or name.startswith(('openshift', 'kube-'))): + raise Failure('dedicated projection names required') + if config['namespace'] == config['targetNamespace']: + raise Failure('proxy and runtime namespaces must differ') + try: + identity = uuid.UUID(config['ownerID']) + if str(identity) != config['ownerID'] or identity.int == 0: + raise ValueError() + except (ValueError, TypeError, AttributeError): + raise Failure('canonical nonzero installation UUID required') from None + + +def mapping(value): + if not isinstance(value, dict): + raise Failure('invalid projection object') + return value + + +def metadata(obj): + meta = mapping(mapping(obj or {}).get('metadata', {})) + mapping(meta.get('labels', {})) + mapping(meta.get('annotations', {})) + return meta + + +def project(api, config): + validate(config) + owner = config['ownerID'] + for namespace in (config['namespace'], config['targetNamespace']): + obj = api.call('GET', '/api/v1/namespaces/' + namespace) + meta = metadata(obj) + if (meta.get('name') != namespace or meta.get('deletionTimestamp') + or meta.get('labels', {}).get(OWNER) != owner): + raise Failure('namespace ownership mismatch') + name = config['name'] + source = api.call('GET', f"/api/v1/namespaces/{config['namespace']}/configmaps/{name}") + meta = metadata(source) + data = mapping((source or {}).get('data', {})) + if (meta.get('name') != name or meta.get('namespace') != config['namespace'] + or meta.get('deletionTimestamp') or set(data) != {'ca.crt'} + or (source or {}).get('binaryData')): + raise Failure('invalid public CA source') + cert = data['ca.crt'] + if (not isinstance(cert, str) or len(cert) > 32768 or 'PRIVATE KEY' in cert + or not re.fullmatch(r'-----BEGIN CERTIFICATE-----\s+[A-Za-z0-9+/=\s]+-----END CERTIFICATE-----\s*', cert)): + raise Failure('public certificate only') + try: + ssl.create_default_context().load_verify_locations(cadata=cert) + except (ValueError, ssl.SSLError): + raise Failure('invalid public certificate') from None + base = f"/api/v1/namespaces/{config['targetNamespace']}/configmaps" + existing = api.call('GET', base + '/' + name) + if existing is None: + api.call('POST', base, {'apiVersion': 'v1', 'kind': 'ConfigMap', + 'metadata': {'name': name, 'namespace': config['targetNamespace'], + 'labels': {OWNER: owner}, + 'annotations': {'argocd.argoproj.io/sync-options': KEEP}}, + 'immutable': True, 'data': {'ca.crt': cert}}) + existing = api.call('GET', base + '/' + name) + meta = metadata(existing) + if (meta.get('name') != name or meta.get('namespace') != config['targetNamespace'] + or meta.get('deletionTimestamp') or meta.get('ownerReferences') + or meta.get('labels', {}).get(OWNER) != owner + or meta.get('annotations', {}).get('argocd.argoproj.io/sync-options') != KEEP + or (existing or {}).get('immutable') is not True + or (existing or {}).get('binaryData') or (existing or {}).get('data') != {'ca.crt': cert}): + raise Failure('public projection differs; explicit trust recovery required') + + +if __name__ == '__main__': + try: + project(Api(), json.loads(Path('/config/config.json').read_text())) + print('Public proxy CA projection verified') + except (Failure, OSError, ValueError) as exc: + print(str(exc) if isinstance(exc, Failure) else 'projection configuration unavailable', file=sys.stderr) + sys.exit(1) diff --git a/charts/saw-kubernetes-proxy/templates/proxy.yaml b/charts/saw-kubernetes-proxy/templates/proxy.yaml new file mode 100644 index 000000000..ac074fa66 --- /dev/null +++ b/charts/saw-kubernetes-proxy/templates/proxy.yaml @@ -0,0 +1,226 @@ +{{- if .Values.enabled }} +{{- $v := .Values -}} +{{- range $name := list .Release.Namespace $v.targetNamespace $v.gateway }} +{{- if or (gt (len $name) 40) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $name)) (eq $name "default") (hasPrefix "openshift" $name) (hasPrefix "kube-" $name) }}{{- fail "dedicated DNS-label names of at most 40 characters required" }}{{- end }}{{- end }} +{{- if eq .Release.Namespace $v.targetNamespace }}{{- fail "trusted and runtime namespaces must differ" }}{{- end }} +{{- if or (eq $v.ownerID "00000000-0000-0000-0000-000000000000") (not (regexMatch "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" $v.ownerID)) }}{{- fail "installation UUID required" }}{{- end }} +{{- range $image := list $v.image $v.helperImage }}{{- if not (regexMatch "^.+@sha256:[0-9a-f]{64}$" $image) }}{{- fail "digest-pinned images required" }}{{- end }}{{- end }} +{{- if or (not $v.policyConfigMap) (not $v.credentialSecretNames) (not $v.network.bootstrapEgress) (not $v.network.proxyEgress) }}{{- fail "existing policy, credential references and explicit egress required" }}{{- end }} +{{- range $name := append (deepCopy $v.credentialSecretNames) $v.policyConfigMap }}{{- if or (gt (len $name) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $name)) }}{{- fail "policy and credential references must be DNS labels" }}{{- end }}{{- end }} +{{- if ne (len (uniq $v.credentialSecretNames)) (len $v.credentialSecretNames) }}{{- fail "duplicate credential reference" }}{{- end }} +{{- if not (regexMatch "^[0-9a-f]{64}$" $v.policyDigest) }}{{- fail "SHA256 of policy config.json required for explicit rollout" }}{{- end }} +{{- $name := printf "%s-credential-proxy" $v.gateway }} +{{- $job := printf "%s-proxy-bootstrap" $v.gateway }} +{{- $ca := printf "%s-proxy-ca" $v.gateway }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $job }} + annotations: {argocd.argoproj.io/sync-wave: "-20"} +data: + config.json: {{ dict "ownerID" $v.ownerID "namespace" .Release.Namespace "targetNamespace" $v.targetNamespace "name" $ca | toJson | quote }} + project_ca.py: | +{{ .Files.Get "files/project_ca.py" | indent 4 }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $job }} + annotations: {argocd.argoproj.io/sync-wave: "-20"} +automountServiceAccountToken: false +{{- range $ns := list .Release.Namespace $v.targetNamespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $job }} + namespace: {{ $ns }} + annotations: {argocd.argoproj.io/sync-wave: "-20"} +rules: + - apiGroups: [""] + resources: [configmaps] + verbs: [create] + - apiGroups: [""] + resources: [configmaps] + resourceNames: [{{ $ca }}] + verbs: [get{{ if eq $ns $.Release.Namespace }}, update{{ end }}] +{{- if eq $ns $.Release.Namespace }} + - apiGroups: [""] + resources: [secrets] + verbs: [create] + - apiGroups: [""] + resources: [secrets] + resourceNames: [{{ $ca }}] + verbs: [get] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $job }} + namespace: {{ $ns }} + annotations: {argocd.argoproj.io/sync-wave: "-20"} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $job }}} +subjects: + - {kind: ServiceAccount, name: {{ $job }}, namespace: {{ $.Release.Namespace }}} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ $job }} + annotations: {argocd.argoproj.io/sync-wave: "-20"} +rules: + - apiGroups: [""] + resources: [namespaces] + resourceNames: [{{ .Release.Namespace }}, {{ $v.targetNamespace }}] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ $job }} + annotations: {argocd.argoproj.io/sync-wave: "-20"} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ .Release.Namespace }}-{{ $job }}} +subjects: + - {kind: ServiceAccount, name: {{ $job }}, namespace: {{ .Release.Namespace }}} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $job }} + annotations: {argocd.argoproj.io/sync-wave: "-20"} +spec: + podSelector: + matchLabels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: proxy-bootstrap} + policyTypes: [Ingress, Egress] + ingress: [] + egress: +{{ toYaml $v.network.bootstrapEgress | indent 4 }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $job }} + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded + argocd.argoproj.io/sync-wave: "-10" +spec: + backoffLimit: 2 + activeDeadlineSeconds: 300 + template: + metadata: + labels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: proxy-bootstrap} + spec: + serviceAccountName: {{ $job }} + automountServiceAccountToken: false + restartPolicy: Never + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + initContainers: + - name: stock-ca + image: {{ $v.image }} + args: [init-ca] + env: + - {name: POD_NAMESPACE, value: {{ .Release.Namespace }}} + - {name: SAW_CA_SECRET_NAME, value: {{ $ca }}} + - {name: SAW_CA_CONFIGMAP_NAME, value: {{ $ca }}} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + resources: {requests: {cpu: 50m, memory: 64Mi}, limits: {memory: 256Mi}} + volumeMounts: + - {name: api, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true} + containers: + - name: public-projection + image: {{ $v.helperImage }} + command: [python3, /config/project_ca.py] + env: [{name: PYTHONDONTWRITEBYTECODE, value: "1"}] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + resources: {requests: {cpu: 50m, memory: 64Mi}, limits: {memory: 256Mi}} + volumeMounts: + - {name: api, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true} + - {name: config, mountPath: /config, readOnly: true} + volumes: + - name: config + configMap: {name: {{ $job }}} + - name: api + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: {path: token, expirationSeconds: 600} + - configMap: {name: kube-root-ca.crt, items: [{key: ca.crt, path: ca.crt}]} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $name }}} +automountServiceAccountToken: false +--- +apiVersion: apps/v1 +kind: Deployment +metadata: {name: {{ $name }}} +spec: + replicas: 1 + selector: + matchLabels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: credential-proxy} + template: + metadata: + annotations: {checksum/policy: {{ $v.policyDigest | quote }}} + labels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: credential-proxy} + spec: + serviceAccountName: {{ $name }} + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: proxy + image: {{ $v.image }} + args: [serve] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + resources: {requests: {cpu: 50m, memory: 64Mi}, limits: {memory: 256Mi}} + ports: [{name: connect, containerPort: 3128}, {name: health, containerPort: 8081}] + readinessProbe: {httpGet: {path: /healthz, port: health}, periodSeconds: 5} + livenessProbe: {tcpSocket: {port: connect}, periodSeconds: 30} + volumeMounts: + - {name: policy, mountPath: /etc/saw-proxy, readOnly: true} + - {name: ca, mountPath: /var/run/saw/ca, readOnly: true} +{{- range $i, $secret := $v.credentialSecretNames }} + - {name: credential-{{ $i }}, mountPath: /var/run/saw/secrets/{{ $secret }}, readOnly: true} +{{- end }} + volumes: + - name: policy + configMap: {name: {{ $v.policyConfigMap }}} + - name: ca + secret: {secretName: {{ $ca }}, defaultMode: 0440} +{{- range $i, $secret := $v.credentialSecretNames }} + - name: credential-{{ $i }} + secret: {secretName: {{ $secret }}, defaultMode: 0440} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: {name: {{ $name }}} +spec: + selector: + app.kubernetes.io/instance: {{ $v.gateway }} + openshell.pattern/role: credential-proxy + ports: [{name: connect, port: 3128, targetPort: connect}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $name }}} +spec: + podSelector: + matchLabels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: credential-proxy} + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: {{ $v.targetNamespace }}} + podSelector: + matchLabels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: gateway} + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: {{ $v.targetNamespace }}} + podSelector: + matchLabels: {openshell.ai/gateway-id: {{ $v.gateway }}, openshell.ai/boundary-role: supervisor} + ports: [{protocol: TCP, port: 3128}] + egress: +{{ toYaml $v.network.proxyEgress | indent 4 }} +{{- end }} diff --git a/charts/saw-kubernetes-proxy/values.schema.json b/charts/saw-kubernetes-proxy/values.schema.json new file mode 100644 index 000000000..daa6b9bd3 --- /dev/null +++ b/charts/saw-kubernetes-proxy/values.schema.json @@ -0,0 +1,111 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "ownerID": { + "type": "string" + }, + "targetNamespace": { + "type": "string" + }, + "gateway": { + "type": "string" + }, + "policyConfigMap": { + "type": "string" + }, + "policyDigest": { + "type": "string" + }, + "image": { + "type": "string" + }, + "helperImage": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "credentialSecretNames": { + "type": "array", + "uniqueItems": true, + "maxItems": 16, + "items": { + "type": "string" + } + }, + "network": { + "type": "object", + "additionalProperties": false, + "required": [ + "bootstrapEgress", + "proxyEgress" + ], + "properties": { + "bootstrapEgress": { + "type": "array", + "items": { + "type": "object", + "required": [ + "to", + "ports" + ], + "additionalProperties": false, + "properties": { + "to": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "minProperties": 1 + } + }, + "ports": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "port" + ] + } + } + } + } + }, + "proxyEgress": { + "type": "array", + "items": { + "type": "object", + "required": [ + "to", + "ports" + ], + "additionalProperties": false, + "properties": { + "to": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "minProperties": 1 + } + }, + "ports": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "port" + ] + } + } + } + } + } + } + } + } +} diff --git a/charts/saw-kubernetes-proxy/values.yaml b/charts/saw-kubernetes-proxy/values.yaml new file mode 100644 index 000000000..499e458b2 --- /dev/null +++ b/charts/saw-kubernetes-proxy/values.yaml @@ -0,0 +1,17 @@ +enabled: false +ownerID: "" +targetNamespace: "" +gateway: "" +# Administrator-owned policy and ESO-created credentials in the release namespace. +# No credential values are accepted by this chart. +policyConfigMap: "" +# SHA256 of config.json; changing policy must roll the stock proxy. +policyDigest: "" +credentialSecretNames: [] +image: ghcr.io/validatedpatterns-sandbox/saw-credential-proxy@sha256:73f271de2eb6166315603fbb72c800b939a14770a69c86debaabbd29a46a0f93 +helperImage: registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c +network: + # Standard NetworkPolicy egress rules. Bootstrap needs only API access. + bootstrapEgress: [] + # Explicit DNS and approved upstream paths; the proxy also checks host policy. + proxyEgress: [] diff --git a/charts/saw-portal-container/Chart.yaml b/charts/saw-portal-container/Chart.yaml new file mode 100644 index 000000000..c759f84ed --- /dev/null +++ b/charts/saw-portal-container/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: saw-portal-container +description: Registration-scoped foundation for a portal-managed audited OpenShell workspace +version: 0.1.4 diff --git a/charts/saw-portal-container/files/enroll.py b/charts/saw-portal-container/files/enroll.py new file mode 100644 index 000000000..9780d7ad0 --- /dev/null +++ b/charts/saw-portal-container/files/enroll.py @@ -0,0 +1,152 @@ +"""Registration bootstrap: compare reviewed configuration, then publish public, fenced runtime inputs.""" +import json +import os +import ssl +import time +import urllib.error +import urllib.request as h +from pathlib import Path +from saw_kubernetes.adapters import KubernetesResourceVerifier +from saw_kubernetes.audit import capture +from saw_kubernetes.compiler import compile_profile_files, digest +from saw_kubernetes.intent import build_intent +from preflight import PreflightError, api_resources_available, crd_established, verify_kata_platform +from portal_contract import verify_contract +from receipt import receipt_key +from gateway_ca import read_public_ca +from http_errors import format_http_error +from kube_api import request_headers + +S=Path("/var/run/secrets/kubernetes.io/serviceaccount") +class NoRedirect(h.HTTPRedirectHandler): + def redirect_request(self,*args): raise ValueError("redirect") +client=h.build_opener(h.ProxyHandler({}),NoRedirect(),h.HTTPSHandler(context=ssl.create_default_context(cafile=str(S/"ca.crt")))) +def api(path, patch=None): + req=h.Request("https://kubernetes.default.svc"+path,method="PATCH" if patch is not None else "GET", + headers=request_headers((S/"token").read_text().strip(), patch is not None),data=json.dumps(patch).encode() if patch is not None else None) + with client.open(req,timeout=9) as response:return json.load(response) + +def once(x): + u,r,sub,ident=x["user"],x["registration"],x["subject"],x["installation"] + ns,t,hist="saw-"+u,"saw-ctl-"+r,"saw-history-"+r + runtime=x.get("kataRuntimeClass") + mode="kata" if runtime else "container" + def obj(kind,name,namespace=ns): + if kind=="runtimeclass" and runtime and name==runtime["name"]: + return api("/apis/node.k8s.io/v1/runtimeclasses/"+name) + if kind=="namespace":return api("/api/v1/namespaces/"+name) + plural={"persistentvolumeclaim":"persistentvolumeclaims","configmap":"configmaps"}[kind] + return api("/api/v1/namespaces/"+namespace+"/"+plural+"/"+name) + nsobj=obj("namespace",ns)["metadata"] + if nsobj["labels"].get("saw.redhat.com/runtime-owner")!=ident or nsobj.get("deletionTimestamp"):raise ValueError("namespace ownership") + if runtime: + try: + runtime_class=obj("runtimeclass",runtime["name"]) + except urllib.error.HTTPError as exc: + if exc.code == 404:raise PreflightError("RuntimeClassUnavailable") from None + raise + try: + configs=api("/apis/kataconfiguration.openshift.io/v1/kataconfigs")["items"] + except urllib.error.HTTPError as exc: + if exc.code == 404:raise PreflightError("SandboxedContainersUnavailable") from None + raise + nodes=api("/api/v1/nodes")["items"] + try: + cluster_version=api("/apis/config.openshift.io/v1/clusterversions/version") + except urllib.error.HTTPError as exc: + if exc.code == 404:raise PreflightError("ClusterVersionUnavailable") from None + raise + api_readiness={} + for name in ("kataconfigs.kataconfiguration.openshift.io", "sandboxes.agents.x-k8s.io", + "externalsecrets.external-secrets.io", "adminnetworkpolicies.policy.networking.k8s.io"): + try: + api_readiness[name]=crd_established(api("/apis/apiextensions.k8s.io/v1/customresourcedefinitions/"+name)) + except urllib.error.HTTPError as exc: + if exc.code == 404:api_readiness[name]=False + else:raise + try: + admission_discovery=api("/apis/admissionregistration.k8s.io/v1") + except urllib.error.HTTPError as exc: + if exc.code == 404:admission_discovery={} + else:raise + api_readiness["admissionregistration.k8s.io/v1"] = api_resources_available( + admission_discovery, "admissionregistration.k8s.io/v1", + ("validatingadmissionpolicies", "validatingadmissionpolicybindings")) + verify_kata_platform(x, runtime_class, configs, nodes, api_readiness, cluster_version) + cfg=dict(namespace=ns,namespaceUID=nsobj["uid"],ownerID=ident,mode=mode,trustedNamespace=t, + governanceNamespace=x["governance"],gateway=u,proxyPolicy=u+"-proxy-policy",slotKeys=sorted(x["targets"])) + if runtime: cfg["workloadRuntimeClass"]=runtime["name"] + cache={} + def get(p): + if p not in cache:cache[p]=api(p) + return cache[p] + approved=capture(cfg,get)["approved"] + verify_contract(cfg,get,x["reference"]) + profile=obj("configmap","saw-bom-profiles") + if digest(profile["data"])!=x["profileSHA256"]:raise ValueError("unapproved BOM") + def pvc(name,ro): + return dict(claimName=name,uid=obj("persistentvolumeclaim",name)["metadata"]["uid"],readOnly=ro) + harnesses = {} + if "harness-index.yaml" in profile["data"]: + from saw_kubernetes.core import shared + from saw_kubernetes.harness import binding_for, read_bundles + documents, raw, index = shared.split_bom_files(profile["data"]) + bundles = read_bundles(raw, index) + harnesses = {ws.name+"/"+sb.name: binding_for(sb, bundles) + for _, ws in shared.enabled_workspaces(shared.parse_profiles(documents)) + for sb in ws.sandboxes if sb.enabled and sb.harness_ref} + slots={} + for key, target in x["targets"].items(): + base=target["base"] + slot={"workspace":pvc("workspace-"+base,False), + "toolGate":{"ownerLogin":u,"receiptKey":receipt_key(base,hist,"/writer-ca/service-ca.crt"), + "trusted":pvc("saw-gate-"+base,True),"audit":pvc("saw-audit-"+base,False)}, + "resources":x["resources"]} + if x.get("lifecycleVersion") == 1 and target.get("modelInput"): + slot["toolGate"]["modelSelector"] = True + if key in harnesses: + slot["toolGate"]["harness"] = harnesses[key] + if target["ui"]: + host="-".join((u,target["workspace"],target["sandbox"],"ui")) + slot["browser"]={"ownerLogin":u,"origin":"https://"+host+".apps."+x["domain"]} + slots[key]=slot + bindings=dict(ownerID=ident,namespace=ns,namespaceUID=nsobj["uid"],ownerSubject=sub,mode=mode, + gatewayVersion=x["gatewayVersion"],approvedCredentialFreeProviders=[],bindings=slots) + if runtime: + bindings.update(workloadRuntimeClass=runtime["name"],workloadRuntimeClassUID=runtime["uid"], + workloadRuntimeClassDigest=runtime["sha256"]) + if x.get("providerPolicy"): + bindings["proxyBindings"] = [{"secret": b["secret"], "key": b["key"], "type": typ, + "refresh": bool(b.get("refresh"))} + for b in x["providerPolicy"]["bindings"] for typ in b["providers"]] + if x.get("customInferenceURL"): + bindings["customInferenceURL"] = x["customInferenceURL"] + if x.get("modelInputs"): + bindings["modelInputs"]=x["modelInputs"] + plan=compile_profile_files(profile["data"],bindings) + KubernetesResourceVerifier(obj,lambda _:None)(plan) # same snapshot verified above + intent=json.loads(build_intent(profile,bindings,name="saw-runtime-intent",generation=1,pin_plan=True)["data"]["intent.json"]) + public_ca=read_public_ca(t,u,ident,get) + ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=public_ca) + result=json.dumps(dict(registration=r,approved=approved,intent=intent,gatewayCA=public_ca),sort_keys=True) + path=f"/api/v1/namespaces/{t}/configmaps/saw-portal-enrollment" + current=api(path) + if current["metadata"]["labels"].get("saw.redhat.com/runtime-owner")!=ident:raise ValueError("publication ownership") + if current.get("data",{}).get("binding.json",result)!=result:raise ValueError("existing enrollment differs") + # Immutable after publication. Neither the tenant nor the portal has write access. + api(path,{"metadata":{"resourceVersion":current["metadata"]["resourceVersion"]},"immutable":True,"data":{"binding.json":result}}) + print("Published reviewed registration "+r,flush=True) + +if __name__=="__main__": + x=json.loads(Path("/bootstrap/config.json").read_text()) + for n in range(100): + try:once(x);break + except PreflightError as exc: + print("Kata preflight failed: "+exc.reason,flush=True) + raise SystemExit(1) + except Exception as exc: + # Never log an API body, credential, or other input. + detail = format_http_error(exc) if isinstance(exc, urllib.error.HTTPError) else type(exc).__name__ + print("Waiting for reviewed inputs: "+detail+(" "+str(exc)[:120] if type(exc).__name__=="BoundaryError" else ""),flush=True) + if n==99:raise SystemExit(1) + time.sleep(5) diff --git a/charts/saw-portal-container/files/gateway_ca.py b/charts/saw-portal-container/files/gateway_ca.py new file mode 100644 index 000000000..690387ff6 --- /dev/null +++ b/charts/saw-portal-container/files/gateway_ca.py @@ -0,0 +1,43 @@ +"""Read the owner-bound public gateway CA from its workspace namespace.""" +import re +import ssl + + +NAME = re.compile(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?\Z") +OWNER = "saw.redhat.com/runtime-owner" +INVALID = "public gateway CA is missing, foreign or malformed" + + +def configmap_path(namespace, gateway): + if any(not isinstance(value, str) or not NAME.fullmatch(value) + for value in (namespace, gateway)): + raise ValueError(INVALID) + return f"/api/v1/namespaces/{namespace}/configmaps/{gateway}-gateway-ca" + + +def read_public_ca(namespace, gateway, owner_id, get): + """Read one immutable public CA ConfigMap. Never accept an unowned map.""" + obj = get(configmap_path(namespace, gateway)) + if not isinstance(obj, dict): + raise ValueError(INVALID) + metadata, data = obj.get("metadata"), obj.get("data") + if not isinstance(metadata, dict) or not isinstance(data, dict): + raise ValueError(INVALID) + labels = metadata.get("labels") + ca = data.get("ca.crt") + if (not isinstance(labels, dict) or obj.get("apiVersion") != "v1" + or obj.get("kind") != "ConfigMap" + or metadata.get("name") != gateway + "-gateway-ca" + or metadata.get("namespace") != namespace + or labels.get(OWNER) != owner_id + or metadata.get("deletionTimestamp") or metadata.get("ownerReferences") + or obj.get("immutable") is not True + or set(data) != {"ca.crt"} + or not isinstance(ca, str) + or not re.fullmatch(r"-----BEGIN CERTIFICATE-----\s+[A-Za-z0-9+/=\s]+-----END CERTIFICATE-----\s*", ca)): + raise ValueError(INVALID) + try: + ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=ca) + except (ssl.SSLError, ValueError): + raise ValueError(INVALID) from None + return ca diff --git a/charts/saw-portal-container/files/http_errors.py b/charts/saw-portal-container/files/http_errors.py new file mode 100644 index 000000000..4131cabda --- /dev/null +++ b/charts/saw-portal-container/files/http_errors.py @@ -0,0 +1,28 @@ +"""Format Kubernetes API errors without exposing response data.""" +from urllib.parse import urlsplit + + +def format_http_error(error): + """Return only the status and Kubernetes API path for an HTTP error.""" + status = getattr(error, "code", None) + if not isinstance(status, int) or isinstance(status, bool) or not 100 <= status <= 599: + return "HTTPError" + + detail = f"HTTP {status}" + url = getattr(error, "url", None) + if not isinstance(url, str): + return detail + + try: + parsed = urlsplit(url) + if parsed.scheme != "https" or parsed.hostname != "kubernetes.default.svc": + return detail + except ValueError: + return detail + + path = parsed.path + if not path.startswith(("/api/", "/apis/")): + return detail + if any(ord(char) < 32 or ord(char) == 127 for char in path): + return detail + return f"{detail} {path}" diff --git a/charts/saw-portal-container/files/kube_api.py b/charts/saw-portal-container/files/kube_api.py new file mode 100644 index 000000000..0ce856015 --- /dev/null +++ b/charts/saw-portal-container/files/kube_api.py @@ -0,0 +1,11 @@ +"""Build safe JSON headers for Kubernetes API requests.""" + + +def request_headers(token, patch=False): + headers = { + "Authorization": "Bearer " + token, + "Accept": "application/json", + } + if patch: + headers["Content-Type"] = "application/merge-patch+json" + return headers diff --git a/charts/saw-portal-container/files/portal_contract.py b/charts/saw-portal-container/files/portal_contract.py new file mode 100644 index 000000000..94cd4e791 --- /dev/null +++ b/charts/saw-portal-container/files/portal_contract.py @@ -0,0 +1,80 @@ +"""Reusable owner-specific approval of a previously reviewed OpenShift boundary. + +Grant no permissions here. Enrollment runs with named, read-only API access. +The normal runtime verifier still pins object UIDs and the full, unmodified +resource hashes after each registration; this comparison is the prerequisite. +""" +import copy +import hashlib +import ipaddress +import json +import re +import ssl +from saw_kubernetes.boundary import BoundaryError, fingerprint, objects + + +def contract(config, get): + ns, trusted, owner = config["namespace"], config["trustedNamespace"], config["ownerID"] + if not re.fullmatch(r"saw-ctl-[0-9a-f]{16}", trusted): + raise BoundaryError("InvalidControlNamespace") + output = {} + for key,(path,field) in objects(config).items(): + obj=get(path) + value=copy.deepcopy(obj[field]) + if key.startswith("governance"): + # Cluster installation is shared across registrations. Keep both UID and hash. + output[key]={"uid":obj["metadata"]["uid"],"sha256":fingerprint(value)} + continue + if key=="publicCA": + source=get(f"/api/v1/namespaces/{trusted}/configmaps/{config['gateway']}-proxy-ca") + if obj.get("metadata",{}).get("labels",{}).get("saw.redhat.com/runtime-owner")!=owner or set(value)!={"ca.crt"} or value!=source.get("data") or source["metadata"].get("deletionTimestamp"): + raise BoundaryError("UntrustedProxyCA") + cert=value["ca.crt"] + if not re.fullmatch(r"-----BEGIN CERTIFICATE-----\s+[A-Za-z0-9+/=\s]+-----END CERTIFICATE-----\s*",cert): + raise BoundaryError("InvalidProxyCA") + try: + ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=cert) + except (ssl.SSLError,ValueError) as exc: + raise BoundaryError("InvalidProxyCA") from exc + output[key]="equals independent trusted proxy public CA" + continue + if key=="gatewayService": + ips=value.get("clusterIPs") + if (value.get("type")!="ClusterIP" or not isinstance(ips,list) or not ips or value.get("clusterIP")!=ips[0] + or [("IPv4" if ipaddress.ip_address(ip).version==4 else "IPv6") for ip in ips]!=value.get("ipFamilies")): + raise BoundaryError("InvalidServiceAllocation") + # Only API-assigned addresses vary. Exact ports, selectors, type, + # ipFamilies, traffic policies and all remaining fields are pinned. + del value["clusterIP"],value["clusterIPs"] + if key=="gateway": + annotations=value["template"]["metadata"].get("annotations",{}) + if not re.fullmatch("[0-9a-f]{64}",annotations.get("checksum/config","")): + raise BoundaryError("MissingConfigRolloutMarker") + # The config itself is approved under gatewayConfig. + del annotations["checksum/config"] + text=json.dumps(value,sort_keys=True,separators=(",",":")) + if "@SAW_" in text: + raise BoundaryError("ReservedApprovalMarker") + # Only substitutions produced by this owner's template are accepted. + # The runtime namespace, owner login and OIDC issuer remain literal. + text=text.replace(trusted,"@SAW_CONTROL_NS@").replace(owner,"@SAW_INSTALLATION@") + if key == "validation": + slots = config.get("slotKeys", []) + registration = trusted.removeprefix("saw-ctl-") + for slot in slots: + relay = hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] if len(slots) > 1 else registration + # Substitute only registered name forms. Keep slot, CA and spec + # checks literal, including the association between Job and PVC. + if len(slots) > 1: + text = text.replace(config["gateway"] + "-" + relay[:8], "@SAW_SLOT_" + slot + "@") + text = text.replace("gate-" + relay + "-", "gate-@SAW_RELAY_" + slot + "@-") + text = text.replace("ui-" + relay + "-browser-proof", "ui-@SAW_RELAY_" + slot + "@-browser-proof") + output[key]=fingerprint(json.loads(text)) + return output + + +def verify_contract(config, get, expected): + observed=contract(config,get) + for key in set(observed)|set(expected): + if observed.get(key)!=expected.get(key): + raise BoundaryError("UnapprovedRegistrationResource:"+key) diff --git a/charts/saw-portal-container/files/preflight.py b/charts/saw-portal-container/files/preflight.py new file mode 100644 index 000000000..c32862180 --- /dev/null +++ b/charts/saw-portal-container/files/preflight.py @@ -0,0 +1,477 @@ +"""Shared, read-only checks for SAW Kubernetes workspace provisioning. + +Callers collect facts from their cluster client. This module validates only +public configuration and observed resource metadata. It never reads Secrets. +""" + +import hashlib +import json +import re + + +class PreflightError(ValueError): + """A failed check with a fixed, safe status code.""" + + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +CHECK_REASONS = { + "api": "ApiUnavailable", + "version": "ClusterVersionUnavailable", + "osc": "SandboxedContainersUnavailable", + "kataConfig": "KataConfigNotReady", + "workers": "KataWorkersNotReady", + "runtimeClass": "RuntimeClassRejected", + "agentSandbox": "AgentSandboxUnavailable", + "admission": "AdmissionUnavailable", + "network": "NetworkPolicyRejected", + "argo": "ArgoAccessDenied", + "storage": "StorageUnavailable", + "secrets": "SecretServicesUnavailable", + "registry": "RegistryUnavailable", + "providerEgress": "ProviderEgressRejected", + "revisions": "UnpinnedRevision", +} + +COMMON_CHECKS = ( + "api", "version", "agentSandbox", "admission", "network", "argo", + "storage", "secrets", "registry", "providerEgress", "revisions", +) +KATA_CHECKS = ("osc", "kataConfig", "workers", "runtimeClass") + +# This pinned image has /sandbox mode 0770, owned by 1000:0. The OpenShift +# namespace UID cannot traverse it. The reviewed notebook build adds o+x. +NON_TRAVERSABLE_WORKLOAD_IMAGES = frozenset({ + "quay.io/aipcc/base-images/agentic/openclaw@sha256:" + "896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", +}) + + +def verify_profile_image_access(site, profile_name, catalog): + """Reject known inaccessible workload images before enrollment starts.""" + profile = catalog.get(profile_name) if isinstance(catalog, dict) else None + if not isinstance(profile, dict) or not isinstance(site, dict): + raise PreflightError("WorkloadImageApprovalMissing") + overrides = site.get("profileImageOverrides", {}) + if not isinstance(overrides, dict): + raise PreflightError("WorkloadImageApprovalInvalid") + selected = overrides.get(profile_name, {}) + if not isinstance(selected, dict): + raise PreflightError("WorkloadImageApprovalInvalid") + for workspace in profile.get("workspaces", []): + if not isinstance(workspace, dict) or not workspace.get("enabled"): + continue + for sandbox in workspace.get("sandboxes", []): + if not isinstance(sandbox, dict) or not sandbox.get("enabled"): + continue + image = sandbox.get("image") + if image in NON_TRAVERSABLE_WORKLOAD_IMAGES: + replacement = selected.get(sandbox.get("name"), image) + if replacement == image: + raise PreflightError("WorkloadImageNotNamespaceAccessible") + if (not isinstance(replacement, str) or not re.fullmatch( + r"[a-z0-9][a-z0-9./:_-]+@sha256:[a-f0-9]{64}", replacement)): + raise PreflightError("WorkloadImageApprovalInvalid") + return True + + +def runtime_placement(runtime_class): + """Return the canonical placement payload used by owner approvals.""" + return {key: runtime_class.get(key) for key in ("handler", "overhead", "scheduling")} + + +def runtime_placement_digest(runtime_class): + raw = json.dumps(runtime_placement(runtime_class), sort_keys=True, + separators=(",", ":"), ensure_ascii=True).encode() + return hashlib.sha256(raw).hexdigest() + + +def fingerprint(value): + """Match the canonical JSON hash used by boundary approvals.""" + raw = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(raw.encode()).hexdigest() + + +def validation_policy_digest(spec, owner, registration, installation_id, slot_keys): + """Hash a live workspace policy with only registration names normalized.""" + if (not isinstance(spec, dict) or not isinstance(owner, str) or not owner + or not isinstance(registration, str) or not re.fullmatch(r"[a-f0-9]{16}", registration) + or not isinstance(installation_id, str) or not installation_id + or not isinstance(slot_keys, (list, tuple)) or not slot_keys + or any(not isinstance(slot, str) or not slot for slot in slot_keys) + or len(set(slot_keys)) != len(slot_keys)): + raise PreflightError("BoundaryApprovalInvalid") + text = json.dumps(spec, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + if "@SAW_" in text: + raise PreflightError("BoundaryApprovalInvalid") + text = text.replace("saw-ctl-" + registration, "@SAW_CONTROL_NS@") + text = text.replace(installation_id, "@SAW_INSTALLATION@") + for slot in slot_keys: + relay = (hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] + if len(slot_keys) > 1 else registration) + if len(slot_keys) > 1: + text = text.replace(owner + "-" + relay[:8], "@SAW_SLOT_" + slot + "@") + text = text.replace("gate-" + relay + "-", "gate-@SAW_RELAY_" + slot + "@-") + text = text.replace("ui-" + relay + "-browser-proof", + "ui-@SAW_RELAY_" + slot + "@-browser-proof") + return fingerprint(json.loads(text)) + + +def verify_boundary_hash(reference, key, observed): + """Fail with a fixed reason when a live policy differs from approval.""" + if not isinstance(reference, dict): + raise PreflightError("BoundaryApprovalMissing") + if key not in reference: + raise PreflightError("BoundaryApprovalMissing") + approved = reference.get(key) + if not isinstance(approved, str) or not re.fullmatch(r"[a-f0-9]{64}", approved): + raise PreflightError("BoundaryApprovalInvalid") + if approved != observed: + raise PreflightError("BoundaryApprovalChanged") + return True + + +def verify_boundary_reference(reference, mode): + """Require every policy approval before a new workspace can be reserved.""" + if mode not in {"container", "kata"}: + raise PreflightError("BoundaryApprovalInvalid") + if not isinstance(reference, dict): + raise PreflightError("BoundaryApprovalMissing") + keys = {"validation", "binding"} + if mode == "kata": + keys.update(("sandboxPlacement", "sandboxPlacementBinding")) + for key in keys: + approved = reference.get(key) + if approved is None: + raise PreflightError("BoundaryApprovalMissing") + if not isinstance(approved, str) or not re.fullmatch(r"[a-f0-9]{64}", approved): + raise PreflightError("BoundaryApprovalInvalid") + return True + + +def check_runtime_class(approved, observed): + """Check a RuntimeClass and return fixed, safe failure reasons.""" + if not isinstance(approved, dict) or not approved.get("name"): + raise PreflightError("RuntimeClassApprovalMissing") + if not isinstance(observed, dict): + raise PreflightError("RuntimeClassUnavailable") + metadata = observed.get("metadata") or {} + if not isinstance(metadata, dict): + raise PreflightError("RuntimeClassUnavailable") + if metadata.get("name") != approved["name"]: + raise PreflightError("RuntimeClassNameMismatch") + if not approved.get("uid") or metadata.get("uid") != approved["uid"]: + raise PreflightError("RuntimeClassIdentityMismatch") + if observed.get("handler") != "kata": + raise PreflightError("RuntimeClassHandlerMismatch") + labels = metadata.get("labels") or {} + if not isinstance(labels, dict): + raise PreflightError("RuntimeClassNotAttachable") + if labels.get("openshell.ai/sandbox-attachable") != "true": + raise PreflightError("RuntimeClassNotAttachable") + if not isinstance(observed.get("scheduling"), dict) or not observed["scheduling"]: + raise PreflightError("RuntimeClassSchedulingInvalid") + approved_digest = approved.get("sha256") + if not isinstance(approved_digest, str) or not re.fullmatch(r"[a-f0-9]{64}", approved_digest): + raise PreflightError("RuntimeClassApprovalInvalid") + if runtime_placement_digest(observed) != approved_digest: + raise PreflightError("RuntimeClassPlacementChanged") + return True + + +def check_observations(mode, observations): + """Require a complete fact set. Missing facts fail closed with safe codes. + + Values are booleans produced by a read-only cluster adapter. Do not pass + API responses, exception strings, URLs with credentials, or Secret data. + """ + if mode not in {"container", "kata"} or not isinstance(observations, dict): + raise PreflightError("InvalidPreflightRequest") + required = COMMON_CHECKS + (KATA_CHECKS if mode == "kata" else ()) + for check in required: + if observations.get(check) is not True: + raise PreflightError(CHECK_REASONS[check]) + return tuple(required) + + +def configuration_facts(site): + """Check site-owned inputs that do not require cluster access.""" + if not isinstance(site, dict): + return {key: False for key in ("network", "storage", "secrets", "registry", + "providerEgress", "revisions")} + stack = site.get("stack") if isinstance(site.get("stack"), dict) else {} + revisions = [item.get("revision") for item in stack.values() if isinstance(item, dict)] + required_components = ("identity", "policy", "proxy", "gateway", "audit") + revision_pin = all(isinstance(stack.get(name), dict) for name in required_components) and all( + isinstance(stack[name].get("revision"), str) + and re.fullmatch(r"[a-f0-9]{40}", stack[name]["revision"]) + for name in required_components) and bool(revisions) and len(revisions) == len(stack) and all( + isinstance(value, str) and re.fullmatch(r"[a-f0-9]{40}", value) for value in revisions) + enrollment = site.get("automaticEnrollment") or {} + if enrollment and not isinstance(enrollment, dict): + revision_pin = False + elif enrollment: + revision_pin = revision_pin and bool(re.fullmatch(r"[a-f0-9]{40}", str(enrollment.get("revision", "")))) + + image_values = [] + storage_values = [] + def walk(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + next_path = path + (str(key).lower(),) + if key == "image" and isinstance(child, str) and child: + image_values.append(child) + if key in {"storageClass", "storageClassName"} and isinstance(child, str) and child: + storage_values.append(child) + if key == "className" and "storage" in path and isinstance(child, str) and child: + storage_values.append(child) + walk(child, next_path) + elif isinstance(value, list): + for child in value: + walk(child, path) + walk(site) + + declared_hosts = site.get("providerAllowHosts") + policy = site.get("providerPolicy") + clean_hosts = (isinstance(declared_hosts, list) and bool(declared_hosts) + and all(isinstance(host, str) and host and "*" not in host + for host in declared_hosts) + and (policy is None or (isinstance(policy, dict) + and isinstance(policy.get("allowHosts"), list) + and all(isinstance(host, str) for host in policy["allowHosts"]) + and set(policy["allowHosts"]) == set(declared_hosts)))) + proxy = stack.get("proxy") or {} + proxy_values = proxy.get("values") or {} if isinstance(proxy, dict) else {} + bootstrap_egress = ((proxy_values.get("network") or {}).get("bootstrapEgress") + if isinstance(proxy_values, dict) else None) + storage_ok = bool(storage_values) and all(value.strip() for value in storage_values) + return { + "network": bool(stack.get("policy")) and isinstance(bootstrap_egress, list) + and bool(bootstrap_egress), + "storage": storage_ok, + "secrets": isinstance(site.get("secretStore", "vault-backend"), str) + and bool(site.get("secretStore", "vault-backend")), + "registry": bool(image_values) and all("@sha256:" in image for image in image_values), + "providerEgress": clean_hosts, + "revisions": revision_pin, + } + + +def storage_class_names(site): + names = set() + def walk(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + if key in {"storageClass", "storageClassName"} and isinstance(child, str) and child: + names.add(child) + elif key == "className" and "storage" in path and isinstance(child, str) and child: + names.add(child) + walk(child, path + (str(key).lower(),)) + elif isinstance(value, list): + for child in value: + walk(child, path) + walk(site) + return tuple(sorted(names)) + + +def verify_storage_classes(site, storage_classes): + names = storage_class_names(site) + available = {item.get("metadata", {}).get("name") for item in storage_classes + if isinstance(item, dict)} + default = any((item.get("metadata", {}).get("annotations") or {}).get( + "storageclass.kubernetes.io/is-default-class") == "true" + for item in storage_classes if isinstance(item, dict)) + if names and not set(names).issubset(available): + raise PreflightError("StorageUnavailable") + if not names and not default: + raise PreflightError("StorageUnavailable") + return names + + +def verify_secret_store(site, stores): + name = secret_store_name(site) + for store in stores: + metadata = store.get("metadata") or {} if isinstance(store, dict) else {} + if isinstance(metadata, dict) and metadata.get("name") == name and _ready(store): + return name + raise PreflightError("SecretServicesUnavailable") + + +def secret_store_name(site): + if not isinstance(site, dict): + return "vault-backend" + stack = site.get("stack") if isinstance(site.get("stack"), dict) else {} + identity = stack.get("identity") if isinstance(stack.get("identity"), dict) else {} + identity_values = identity.get("values") if isinstance(identity.get("values"), dict) else {} + return site.get("secretStore") or identity_values.get("secretStore") or "vault-backend" + + +def _ready(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and any( + isinstance(c, dict) and c.get("type") in {"Ready", "Completed"} and c.get("status") == "True" + for c in conditions) + + +def kata_config_ready(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + nodes = status.get("kataNodes") or {} + if not isinstance(nodes, dict): + return False + count = nodes.get("nodeCount") + ready_count = nodes.get("readyNodeCount") + if (type(count) is not int or count < 1 or type(ready_count) is not int + or ready_count != count): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and not any( + isinstance(condition, dict) and condition.get("type") == "InProgress" + and condition.get("status") == "True" for condition in conditions) + + +def kata_config_installed_nodes(obj): + if not isinstance(obj, dict): + return () + status = obj.get("status") or {} + if not isinstance(status, dict): + return () + nodes = status.get("kataNodes") or {} + if not isinstance(nodes, dict): + return () + installed = nodes.get("installed") + if not isinstance(installed, list): + return () + return tuple(name for name in installed if isinstance(name, str) and name) + + +def crd_established(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and any( + isinstance(condition, dict) and condition.get("type") == "Established" + and condition.get("status") == "True" for condition in conditions) + + +def api_resources_available(discovery, group_version, resource_names): + """Check served API resources from discovery, including built-in APIs.""" + if not isinstance(discovery, dict) or discovery.get("groupVersion") != group_version: + return False + resources = discovery.get("resources") + if not isinstance(resources, list): + return False + available = {item.get("name") for item in resources if isinstance(item, dict)} + return set(resource_names).issubset(available) + + +def cluster_version_ready(obj): + try: + verify_cluster_version(obj) + return True + except PreflightError: + return False + + +def verify_cluster_version(obj): + if not isinstance(obj, dict): + raise PreflightError("ClusterVersionUnavailable") + status = obj.get("status") or {} + if not isinstance(status, dict): + raise PreflightError("ClusterVersionUnavailable") + conditions = status.get("conditions") or [] + available = isinstance(conditions, list) and any( + isinstance(condition, dict) and condition.get("type") == "Available" + and condition.get("status") == "True" for condition in conditions) + if not available: + raise PreflightError("ClusterVersionUnavailable") + desired = status.get("desired") or {} + version = desired.get("version") if isinstance(desired, dict) else None + if not isinstance(version, str): + raise PreflightError("ClusterVersionUnavailable") + version = version.strip() + match = re.fullmatch(r"(\d+)\.(\d+)(?:\.\d+)?(?:-[A-Za-z0-9.-]+)?", version) + if not match: + raise PreflightError("ClusterVersionUnavailable") + if tuple(map(int, match.groups())) < (4, 22): + raise PreflightError("UnsupportedOpenShiftVersion") + return version + + +def vault_health_ready(obj): + return isinstance(obj, dict) and obj.get("initialized") is True and obj.get("sealed") is False + + +def _matches_selector(labels, selector): + return bool(selector) and all(labels.get(key) == value for key, value in selector.items()) + + +def verify_kata_platform(site, runtime_class, kata_configs, nodes, api_readiness, + cluster_version): + """Check live Kata prerequisites from read-only Kubernetes API snapshots.""" + verify_cluster_version(cluster_version) + approved = site.get("kataRuntimeClass") if isinstance(site, dict) else None + check_runtime_class(approved, runtime_class) + if not isinstance(api_readiness, dict): + raise PreflightError("AgentSandboxUnavailable") + if api_readiness.get("kataconfigs.kataconfiguration.openshift.io") is not True: + raise PreflightError("SandboxedContainersUnavailable") + if not isinstance(kata_configs, list): + raise PreflightError("KataConfigNotReady") + ready_configs = [item for item in kata_configs + if isinstance(item, dict) and kata_config_ready(item)] + if not ready_configs: + raise PreflightError("KataConfigNotReady") + installed_nodes = {name for item in ready_configs for name in kata_config_installed_nodes(item)} + placement = (runtime_class.get("scheduling") or {}).get("nodeSelector") or {} + if not isinstance(placement, dict) or not placement: + raise PreflightError("RuntimeClassSchedulingInvalid") + eligible = [] + if not isinstance(nodes, list): + raise PreflightError("KataWorkersNotReady") + for node in nodes: + if not isinstance(node, dict): + continue + metadata = node.get("metadata") or {} + if not isinstance(metadata, dict): + continue + labels = metadata.get("labels") or {} + if not isinstance(labels, dict): + continue + roles = {key.removeprefix("node-role.kubernetes.io/") for key in labels + if key.startswith("node-role.kubernetes.io/")} + status = node.get("status") or {} + conditions = status.get("conditions", []) if isinstance(status, dict) else [] + ready = isinstance(conditions, list) and any( + isinstance(c, dict) and c.get("type") == "Ready" and c.get("status") == "True" + for c in conditions) + spec = node.get("spec") or {} + if (ready and isinstance(spec, dict) and not spec.get("unschedulable") + and labels.get("kubernetes.io/arch") == "amd64" + and "worker" in roles and not roles.intersection({"master", "control-plane"}) + and _matches_selector(labels, placement)): + eligible.append(metadata.get("name")) + if not eligible or not set(eligible).intersection(installed_nodes): + raise PreflightError("KataWorkersNotReady") + if api_readiness.get("sandboxes.agents.x-k8s.io") is not True: + raise PreflightError("AgentSandboxUnavailable") + if api_readiness.get("externalsecrets.external-secrets.io") is not True: + raise PreflightError("SecretServicesUnavailable") + if api_readiness.get("adminnetworkpolicies.policy.networking.k8s.io") is not True: + raise PreflightError("NetworkPolicyRejected") + if api_readiness.get("admissionregistration.k8s.io/v1") is not True: + raise PreflightError("AdmissionUnavailable") + return tuple(sorted(eligible)) diff --git a/charts/saw-portal-container/files/receipt.py b/charts/saw-portal-container/files/receipt.py new file mode 100644 index 000000000..2505f5be9 --- /dev/null +++ b/charts/saw-portal-container/files/receipt.py @@ -0,0 +1,22 @@ +"""Fetch only the writer's Ed25519 public receipt key using its service identity.""" +import base64 +import re +import ssl +import urllib.request + + +def receipt_key(service, namespace, ca_file, *, opener=None): + label = r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?" + if not all(re.fullmatch(label, s) for s in (service, namespace)): + raise ValueError("invalid writer service") + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args): + raise ValueError("writer redirect refused") + opener = opener or urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect(), + urllib.request.HTTPSHandler(context=ssl.create_default_context(cafile=ca_file))) + with opener.open(f"https://{service}.{namespace}.svc:8443/v1/receipt-key", timeout=8) as response: + raw = response.read(65) + der = base64.b64decode(raw, validate=True) + if len(der) != 44 or der[:12] != bytes.fromhex("302a300506032b6570032100"): + raise ValueError("expected Ed25519 SubjectPublicKeyInfo") + return raw.decode("ascii") diff --git a/charts/saw-portal-container/templates/_helpers.tpl b/charts/saw-portal-container/templates/_helpers.tpl new file mode 100644 index 000000000..a6ad840c2 --- /dev/null +++ b/charts/saw-portal-container/templates/_helpers.tpl @@ -0,0 +1,13 @@ +{{/* Targets are derived from the selected BOM, never a fixed profile. */}} +{{- define "portal.targets" -}} +{{- $s := .Values.site | default dict -}} +{{ required "BOM-derived sandbox targets are required" $s.targets | toJson }} +{{- end -}} +{{- define "portal.claims" -}} +{{- $claims := list -}} +{{- range $slot := include "portal.targets" . | fromJson -}} +{{- range $prefix := list "workspace-" "saw-gate-" "saw-audit-" -}} +{{- $claims = append $claims (printf "%s%s" $prefix $slot.base) -}} +{{- end -}}{{- end -}} +{{ $claims | toJson }} +{{- end -}} diff --git a/charts/saw-portal-container/templates/auto-enroll.yaml b/charts/saw-portal-container/templates/auto-enroll.yaml new file mode 100644 index 000000000..47853e0fc --- /dev/null +++ b/charts/saw-portal-container/templates/auto-enroll.yaml @@ -0,0 +1,140 @@ +{{- $s := .Values.site | default dict }}{{- $a := $s.automaticEnrollment | default dict }} +{{- if and $a (ne (get $a "enabled" | toString) "false") }} +{{- if or (not (regexMatch "^.+@sha256:[a-f0-9]{64}$" $a.image)) (not (regexMatch "^[a-f0-9]{40}$" $a.revision)) (ne (len $a.reference) (ternary 27 25 (not (empty $s.kataRuntimeClass)))) }}{{ fail "automatic enrollment requires pinned code and a reviewed boundary" }}{{- end }} +{{- $observer := $s.observer | default dict }} +{{- $r := .Values.registration }}{{- $g := .Values.owner }}{{- $n := .Release.Namespace }} +{{- $t := printf "saw-ctl-%s" $r }}{{- $archive := printf "saw-history-%s" $r }} +{{- $targets := include "portal.targets" . | fromJson }} +{{- $bases := list }}{{- range $slot := $targets }}{{- $bases = append $bases $slot.base }}{{- end }} +{{- $gov := $s.governanceNamespace }}{{- $name := printf "enroll-%s" $r }}{{- $policy := printf "saw-%s-runtime" $n }} +{{- $gatewayVersion := dig "stack" "gateway" "values" "kubernetes" "gateway" "version" .Values.driverDefaults.gatewayVersion $s -}} +{{- $bootstrap := dict "registration" $r "installation" .Values.installationID "user" $g "subject" .Values.ownerSubject "governance" $gov "reference" $a.reference "profileSHA256" ($s.selectedDigest | default $a.profileSHA256) "targets" $targets "gatewayVersion" $gatewayVersion "customInferenceURL" ($s.customInferenceURL | default "") "providerPolicy" ($s.providerPolicy | default dict) "lifecycleVersion" ($s.lifecycleVersion | default 0) "modelInputs" ($s.modelInputs | default dict) "resources" $a.resources "domain" .Values.global.clusterDomain -}} +{{- $scripts := (.Files.Glob "files/*.py").AsConfig -}} +{{- with $s.kataRuntimeClass }}{{- $_ := set $bootstrap "kataRuntimeClass" . }}{{- end }} +{{- $revision := dict "chartVersion" .Chart.Version "image" $a.image "config" $bootstrap "scripts" $scripts | toJson | sha256sum | trunc 12 -}} +{{- $scriptName := printf "bootstrap-%s" $revision -}} +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $name }}, namespace: {{ $t }}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: {name: {{ $t }}-{{ $name }}} +rules: + - {apiGroups: [""], resources: [namespaces], resourceNames: [{{ $n }}], verbs: [get]} + - {apiGroups: [""], resources: [nodes], verbs: [list]} + - {apiGroups: [config.openshift.io], resources: [clusterversions], resourceNames: [version], verbs: [get]} + - {apiGroups: [kataconfiguration.openshift.io], resources: [kataconfigs], verbs: [list]} + - {apiGroups: [apiextensions.k8s.io], resources: [customresourcedefinitions], resourceNames: [kataconfigs.kataconfiguration.openshift.io, sandboxes.agents.x-k8s.io, externalsecrets.external-secrets.io, adminnetworkpolicies.policy.networking.k8s.io], verbs: [get]} +{{- with $s.kataRuntimeClass }} + - {apiGroups: [node.k8s.io], resources: [runtimeclasses], resourceNames: [{{ .name | quote }}], verbs: [get]} + - {apiGroups: [admissionregistration.k8s.io], resources: [validatingadmissionpolicies, validatingadmissionpolicybindings], resourceNames: [{{ $policy }}-kata], verbs: [get]} +{{- end }} + - {apiGroups: [policy.networking.k8s.io], resources: [adminnetworkpolicies], verbs: [list]} + - {apiGroups: [policy.networking.k8s.io], resources: [adminnetworkpolicies], resourceNames: [{{ $policy }}-egress], verbs: [get]} + - {apiGroups: [admissionregistration.k8s.io], resources: [validatingadmissionpolicies, validatingadmissionpolicybindings], resourceNames: [{{ $policy }}, {{ $n }}-pods-driver, {{ $n }}-secrets-driver, {{ $n }}-services-driver], verbs: [get]} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: {name: {{ $t }}-{{ $name }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ $t }}-{{ $name }}} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ $t }}}] +{{- range $ns := list $n $t $gov }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $name }}, namespace: {{ $ns }}} +rules: +{{- if eq $ns $n }} + - {apiGroups: [rbac.authorization.k8s.io], resources: [roles], resourceNames: [{{ $g }}-gateway], verbs: [get]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [saw-bom-profiles, {{ $g }}-gateway-config, {{ $g }}-proxy-ca], verbs: [get]} + - {apiGroups: [""], resources: [persistentvolumeclaims], resourceNames: {{ include "portal.claims" $ }}, verbs: [get]} + - {apiGroups: [apps], resources: [statefulsets], resourceNames: [{{ $g }}-gateway], verbs: [get]} + - {apiGroups: [""], resources: [services], resourceNames: [{{ $g }}-gateway], verbs: [get]} + - {apiGroups: [networking.k8s.io], resources: [networkpolicies], resourceNames: [{{ $g }}-gateway, {{ $g }}-default-deny, openshell-sandbox-workloads, openshell-sandbox-supervisors], verbs: [get]} +{{- else if eq $ns $t }} + - {apiGroups: [apps], resources: [deployments], resourceNames: [{{ $g }}-credential-proxy], verbs: [get]} + - {apiGroups: [networking.k8s.io], resources: [networkpolicies], resourceNames: [{{ $g }}-credential-proxy], verbs: [get]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [{{ $g }}-proxy-policy, {{ $g }}-proxy-ca, {{ $g }}-gateway-ca], verbs: [get]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [saw-portal-enrollment], verbs: [get, patch]} +{{- else }} + - {apiGroups: [apps], resources: [deployments], resourceNames: [governance-interceptor], verbs: [get]} + - {apiGroups: [""], resources: [services], resourceNames: [governance-interceptor], verbs: [get]} + - {apiGroups: [""], resources: [configmaps], resourceNames: [governance-interceptor-policy, governance-interceptor-profiles], verbs: [get]} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $name }}, namespace: {{ $ns }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }}} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ $t }}}] +{{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $name }}-pull, namespace: {{ $a.buildNamespace }}} +rules: [{apiGroups: [image.openshift.io], resources: [imagestreams/layers], resourceNames: [{{ base (index (splitList "@" $a.image) 0) | quote }}], verbs: [get]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $name }}-pull, namespace: {{ $a.buildNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }}-pull} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ $t }}}] +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $scriptName }}, namespace: {{ $t }}} +immutable: true +data: + config.json: {{ $bootstrap | toJson | quote }} +{{ $scripts | indent 2 }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: bootstrap-writer, namespace: {{ $archive }}} +spec: + podSelector: {matchExpressions: [{key: saw-audit-writer, operator: In, values: {{ $bases | toJson }}}]} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $t }}}} + podSelector: {matchLabels: {app: {{ $name }}}} + ports: [{port: 8443, protocol: TCP}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: bootstrap-egress, namespace: {{ $t }}} +spec: + podSelector: {matchLabels: {app: {{ $name }}}} + policyTypes: [Egress] + egress: +{{ toYaml $s.stack.proxy.values.network.bootstrapEgress | indent 4 }} + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $archive }}}} + podSelector: {matchExpressions: [{key: saw-audit-writer, operator: In, values: {{ $bases | toJson }}}]} + ports: [{port: 8443, protocol: TCP}] +--- +apiVersion: batch/v1 +kind: Job +metadata: {name: {{ $name }}-{{ $revision }}, namespace: {{ $t }}} +spec: + backoffLimit: 1 + activeDeadlineSeconds: 1200 + template: + metadata: {labels: {app: {{ $name }}}} + spec: + serviceAccountName: {{ $name }} + restartPolicy: Never + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: enroll + image: {{ $a.image | quote }} + command: [python3, /bootstrap/enroll.py] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + volumeMounts: [{name: scripts, mountPath: /bootstrap, readOnly: true},{name: ca, mountPath: /writer-ca, readOnly: true}] + volumes: + - {name: scripts, configMap: {name: {{ $scriptName }}}} + - {name: ca, configMap: {name: openshift-service-ca.crt}} +{{- end }} diff --git a/charts/saw-portal-container/templates/enrolled-runtime.yaml b/charts/saw-portal-container/templates/enrolled-runtime.yaml new file mode 100644 index 000000000..c82c2714d --- /dev/null +++ b/charts/saw-portal-container/templates/enrolled-runtime.yaml @@ -0,0 +1,137 @@ +{{- $site := .Values.site | default dict }} +{{- $e := $site.enrollment | default dict }} +{{- if and $e (eq $e.registration .Values.registration) }} +{{- $reg := .Values.registration }} +{{- $u := .Values.owner }} +{{- $ns := .Release.Namespace }} +{{- $ctl := printf "saw-ctl-%s" $reg }} +{{- $history := printf "saw-history-%s" $reg }} +{{- $b := $e.intent.bindings }} +{{- if or (ne $b.ownerID .Values.installationID) (ne $b.ownerSubject .Values.ownerSubject) (ne $b.namespace $ns) (ne $b.mode (ternary "kata" "container" (not (empty $site.kataRuntimeClass)))) (ne $e.intent.profileRef.name "saw-bom-profiles") }}{{ fail "enrollment belongs to a different registration" }}{{ end }} +{{- if not (regexMatch "^.+@sha256:[0-9a-f]{64}$" $e.image) }}{{ fail "pinned controller image required" }}{{ end }} +{{- $targets := include "portal.targets" . | fromJson }} +{{- $claims := include "portal.claims" . | fromJsonArray }} +{{- if ne (len $b.bindings) (len $targets) }}{{ fail "sandbox slot count mismatch" }}{{- end }} +{{- $intent := dict "schemaVersion" 1 "generation" .Values.intentGeneration "desiredState" ($site.desiredState | default "Running") "bindings" $b "profileRef" $e.intent.profileRef }} +{{- if $site.lifecycleVersion }}{{- $_ := set $intent "restartGeneration" $site.restartGeneration }}{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: {name: saw-runtime-intent, namespace: {{ $ns }}, {{ if $site.lifecycleVersion }}annotations: {argocd.argoproj.io/sync-wave: '3'}, {{ end }}labels: {saw.redhat.com/runtime-owner: {{ $b.ownerID }}}} +data: + intent.json: {{ $intent | toJson | quote }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: {name: portal-ledger, namespace: {{ $ctl }}} +spec: + storageClassName: {{ $site.stack.audit.values.storageClass }} + accessModes: [ReadWriteOnce] + resources: {requests: {storage: 1Gi}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: saw-gate-preparation, namespace: {{ $ns }}} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ printf "portal-gate-%s" $reg }}, namespace: {{ $e.buildNamespace }}} +rules: [{apiGroups: [image.openshift.io], resources: [imagestreams/layers], resourceNames: [{{ base (index (splitList "@" $e.image) 0) | quote }}], verbs: [get]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ printf "portal-gate-%s" $reg }}, namespace: {{ $e.buildNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ printf "portal-gate-%s" $reg }}} +subjects: [{kind: ServiceAccount, name: saw-gate-preparation, namespace: {{ $ns }}}] +{{- range $key,$target := $targets }} +{{- $slot := index $b.bindings $key }} +{{- $gate := $slot.toolGate }} +{{- if or (ne $slot.workspace.claimName (printf "workspace-%s" $target.base)) (ne $gate.trusted.claimName (printf "saw-gate-%s" $target.base)) (ne $gate.audit.claimName (printf "saw-audit-%s" $target.base)) (ne $gate.ownerLogin $u) (not $gate.receiptKey) }}{{ fail "invalid storage binding" }}{{- end }} +{{- if and $target.ui (or (ne $slot.browser.ownerLogin $u) (ne $slot.browser.origin (printf "https://%s-%s-%s-ui.apps.%s" $u $target.workspace $target.sandbox $.Values.global.clusterDomain))) }}{{ fail "invalid owner UI binding" }}{{- end }} +{{- $job := ternary "saw-gate-preparation" (printf "gate-%s" $target.relayID) (eq (len $targets) 1) }} +{{- if $gate.harness }} +{{- $job = printf "%s-h%s" $job (trimPrefix "sha256:" $gate.harness.digest | trunc 12) }} +{{- end }} +{{- $model := "" }} +{{- $publication := max 1 ($site.restartGeneration | default 0) }} +{{- if $gate.modelSelector }} +{{- $model = required "public model selection missing" (index $site.requestedModels $target.modelInput) }} +{{- $job = printf "%s-m%d" $job $publication }} +{{- end }} +{{- /* Job PodSpecs are immutable. A new artifact or bundle gets a new Job, +while an unchanged sync keeps the completed publication. */ -}} +{{- $publicationSpec := dict "version" 2 "image" $e.image "owner" $u "subject" $b.ownerSubject "target" $target "gate" $gate "model" $model "generation" $publication -}} +{{- $job = printf "%s-%s" $job ($publicationSpec | toJson | sha256sum | trunc 10) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: {name: {{ $job }}, namespace: {{ $ns }}, annotations: {argocd.argoproj.io/sync-wave: '2'}} +spec: + backoffLimit: 2 + activeDeadlineSeconds: 360 + template: + spec: + serviceAccountName: saw-gate-preparation + automountServiceAccountToken: false + restartPolicy: Never + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: prep + image: {{ $e.image | quote }} + command: [python3, -m, saw_kubernetes.prepare_gate, --owner, {{ $u | quote }}, --subject, {{ $b.ownerSubject | quote }}, --workspace, {{ $target.workspace }}, --sandbox, {{ $target.sandbox }}, --receipt-key, {{ $gate.receiptKey | quote }}{{ if $target.ui }}, --browser-proof, /proof/relay-proof{{ end }}{{ if $gate.harness }}, --harness, {{ $gate.harness.name | quote }}, {{ $gate.harness.digest | quote }}{{ end }}{{ if $model }}, --model, {{ $model | quote }}, --model-generation, {{ $publication | quote }}{{ end }}] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + volumeMounts: [{name: gate, mountPath: /volume}{{ if $target.ui }}, {name: proof, mountPath: /proof, readOnly: true}{{ end }}{{ if $gate.harness }}, {name: bom, mountPath: /bom, readOnly: true}{{ end }}] + volumes: [{name: gate, persistentVolumeClaim: {claimName: {{ $gate.trusted.claimName }}}}{{ if $target.ui }}, {name: proof, secret: {secretName: ui-{{ $target.relayID }}-browser-proof, items: [{key: relay-proof, path: relay-proof}]}}{{ end }}{{ if $gate.harness }}, {name: bom, configMap: {name: saw-bom-profiles}}{{ end }}] +{{- end }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ printf "saw-c-%s-controller" $reg }} + namespace: {{ .Values.global.vpArgoNamespace }} + annotations: {argocd.argoproj.io/sync-wave: '3'} + labels: {openshell.pattern/owner: {{ $u }}, saw.redhat.com/registration: {{ $reg | quote }}} + finalizers: [resources-finalizer.argocd.argoproj.io/foreground] +spec: + project: {{ $site.project | default "default" }} + destination: {server: 'https://kubernetes.default.svc', namespace: {{ $ctl }}} + source: + repoURL: {{ .Values.global.repoURL | quote }} + targetRevision: {{ $e.revision | quote }} + path: charts/saw-kubernetes-controller + helm: + releaseName: {{ $u }} + valuesObject: + enabled: true + image: {{ $e.image | quote }} + stateClaim: portal-ledger + imageBuildNamespace: {{ $e.buildNamespace }} + claimNames: {{ $claims | toJson }} + configuration: + gateway: {{ $u }} + namespace: {{ $ns }} + trustedNamespace: {{ $ctl }} + governanceNamespace: {{ $site.governanceNamespace }} + proxyPolicy: {{ $u }}-proxy-policy + ownerID: {{ $b.ownerID }} + namespaceUID: {{ $b.namespaceUID }} + mode: {{ $b.mode }} +{{- with $site.kataRuntimeClass }} + workloadRuntimeClass: {{ .name | quote }} +{{- end }} + intent: saw-runtime-intent + profile: saw-bom-profiles + status: saw-runtime-status + approved: {{ $e.approved | toJson }} + trustedEgress: +{{ toYaml $site.stack.proxy.values.network.bootstrapEgress | indent 10 }} + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $ns }}}} + podSelector: {matchLabels: {app.kubernetes.io/instance: {{ $u }}, openshell.pattern/role: gateway}} + ports: [{protocol: TCP, port: 17670}] + ignoreDifferences: + - {group: "", kind: ConfigMap, name: saw-runtime-status, namespace: {{ $ns }}, jsonPointers: [/data]} + syncPolicy: + automated: {prune: true, selfHeal: true} + syncOptions: [RespectIgnoreDifferences=true] +{{- end }} diff --git a/charts/saw-portal-container/templates/foundation.yaml b/charts/saw-portal-container/templates/foundation.yaml new file mode 100644 index 000000000..92a22d8c3 --- /dev/null +++ b/charts/saw-portal-container/templates/foundation.yaml @@ -0,0 +1,87 @@ +{{- $u := required "verified registration owner required" .Values.owner -}} +{{- $id := required "registration required" .Values.registration -}} +{{- if or (gt (len $u) 19) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $u)) (not (regexMatch "^[0-9a-f]{16}$" $id)) }}{{ fail "invalid portal owner or registration" }}{{ end }} +{{- $subject := required "resolved ownerSubject required" .Values.ownerSubject }} +{{- if not (regexMatch "^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$" $subject) }}{{ fail "invalid ownerSubject" }}{{ end }} +{{- $runtime := printf "saw-%s" $u }} +{{- $trusted := printf "saw-ctl-%s" $id }} +{{- if ne .Release.Namespace $runtime }}{{ fail "release namespace must match workspace owner" }}{{ end }} +{{- $prefix := required "owner-scoped vaultPrefix required" .Values.vaultPrefix }} +{{- if not (regexMatch (printf "^[a-zA-Z0-9_-]+/data/([a-zA-Z0-9_-]+/)*saw-%s/%s$" $u $id) $prefix) }}{{ fail "vaultPrefix must match the owner and registration" }}{{ end }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $trusted }} + labels: + saw.redhat.com/runtime-owner: {{ required "installationID required" .Values.installationID | quote }} + saw.redhat.com/portal-owner: {{ $u | quote }} + saw.redhat.com/registration: {{ $id | quote }} + pod-security.kubernetes.io/enforce: restricted + annotations: + argocd.argoproj.io/sync-wave: "-2" +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: default-deny, namespace: {{ $trusted }}, annotations: {argocd.argoproj.io/sync-wave: "-1"}} +spec: {podSelector: {}, policyTypes: [Ingress, Egress], ingress: [], egress: []} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: default-deny, namespace: {{ $runtime }}, annotations: {argocd.argoproj.io/sync-wave: "-1"}} +spec: {podSelector: {}, policyTypes: [Ingress, Egress], ingress: [], egress: []} +{{- $customSecret := "" }} +{{- $sources := dict }} +{{- with required "explicit governed provider policy required" (.Values.site | default dict).providerPolicy }} +{{- $sources = dict }} +{{- range .bindings }} +{{- $fields := get $sources .secret | default list }} +{{- if .refresh }} +{{- range .refresh.material }}{{- $fields = append $fields . }}{{- end }} +{{- else }}{{- $fields = append $fields .key }}{{- end }} +{{- if has "custom-inference" .providers }}{{- $customSecret = .secret }}{{- end }} +{{- $_ := set $sources .secret ($fields | uniq | sortAlpha) }} +{{- end }} +{{- end }} +{{- range $secret, $fields := $sources }} +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ $secret }} + namespace: {{ $trusted }} +spec: + refreshInterval: 1m + secretStoreRef: {kind: ClusterSecretStore, name: {{ $.Values.secretStore | quote }}} + {{- if eq $customSecret $secret }} + target: + name: {{ $secret }} + creationPolicy: Owner + template: + engineVersion: v2 + mergePolicy: Merge + data: + url: {{ required "custom inference requires the enrolled site URL" $.Values.site.customInferenceURL | quote }} + {{- else }} + target: {name: {{ $secret }}, creationPolicy: Owner} + {{- end }} + data: + {{- range $fields }} + - secretKey: {{ . }} + remoteRef: {key: {{ printf "%s/%s" $prefix $secret | quote }}, property: {{ . }}} + {{- end }} +{{- end }} + +{{- $pull := (.Values.site | default dict).imagePull | default dict }} +{{- if $pull }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: portal-image-{{ $id }}, namespace: {{ $pull.namespace }}} +rules: [{apiGroups: [image.openshift.io], resources: [imagestreams/layers], resourceNames: {{ $pull.streams | toJson }}, verbs: [get]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: portal-image-{{ $id }}, namespace: {{ $pull.namespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: portal-image-{{ $id }}} +subjects: [{kind: ServiceAccount, name: {{ $u }}-sandbox, namespace: {{ $runtime }}}] +{{- end }} diff --git a/charts/saw-portal-container/templates/publications.yaml b/charts/saw-portal-container/templates/publications.yaml new file mode 100644 index 000000000..a39f271b2 --- /dev/null +++ b/charts/saw-portal-container/templates/publications.yaml @@ -0,0 +1,29 @@ +{{- $s := .Values.site | default dict }} +{{- if $s.automaticEnrollment }} +{{- $observer := $s.observer | default dict }} +{{- $t := printf "saw-ctl-%s" .Values.registration }} +{{- $targets := include "portal.targets" . | fromJson }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: saw-portal-enrollment, namespace: {{ $t }}, labels: {saw.redhat.com/runtime-owner: {{ .Values.installationID }}}} +{{- $names := list "saw-portal-enrollment" }} +{{- range $slot := $targets }}{{- if $slot.ui }}{{- $names = append $names $slot.subjectMap }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $slot.subjectMap }}, namespace: {{ $t }}, labels: {saw.redhat.com/runtime-owner: {{ $.Values.installationID }}}} +{{- end }}{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: portal-binding-reader, namespace: {{ $t }}} +rules: [{apiGroups: [""], resources: [configmaps], resourceNames: {{ $names | toJson }}, verbs: [get]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: portal-binding-reader, namespace: {{ $t }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: portal-binding-reader} +subjects: [{kind: ServiceAccount, name: {{ $observer.serviceAccount | default "saw-workspaces-generator" }}, namespace: {{ $observer.namespace | default .Values.portalNamespace | default "saw-portal" }}}] +{{- end }} diff --git a/charts/saw-portal-container/templates/relay.yaml b/charts/saw-portal-container/templates/relay.yaml new file mode 100644 index 000000000..5ac8ecea6 --- /dev/null +++ b/charts/saw-portal-container/templates/relay.yaml @@ -0,0 +1,80 @@ +{{- $site := .Values.site | default dict }} +{{- $e := $site.enrollment | default dict }} +{{- if and $e (eq $e.registration .Values.registration) $e.relay }} +{{- $r := $e.relay }} +{{- if not (regexMatch "^[a-f0-9]{40}$" ($r.revision | default "")) }}{{ fail "relay revision must be a full Git commit SHA" }}{{ end }} +{{- $u := .Values.owner }} +{{- $reg := .Values.registration }} +{{- $ctl := printf "saw-ctl-%s" $reg }} +{{- $targets := include "portal.targets" . | fromJson }} +{{- range $target := $targets }}{{- if $target.ui }} +{{- $rid := $target.relayID }} +{{- $ui := printf "saw-ui-%s" $rid }} +{{- $ns := $.Release.Namespace }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: saw-c-{{ $reg }}-{{ ternary "relay" (printf "relay-%s" $rid) (eq (len $targets) 1) }} + namespace: {{ $.Values.global.vpArgoNamespace }} + labels: {openshell.pattern/owner: {{ $u }}, saw.redhat.com/registration: {{ $reg | quote }}} + finalizers: [resources-finalizer.argocd.argoproj.io/foreground] +spec: + project: {{ $site.project | default "default" }} + destination: {server: 'https://kubernetes.default.svc', namespace: {{ $ui }}} + source: + repoURL: {{ $.Values.global.repoURL | quote }} + targetRevision: {{ required "pinned relay revision required" $r.revision | quote }} + path: charts/saw-workspace-relay + helm: + releaseName: {{ printf "ui-%s" $rid }} + valuesObject: + enabled: true + {{- if $site.lifecycleVersion }} + paused: {{ eq $site.desiredState "Stopped" }} + {{- end }} + publishSubject: {{ $site.automaticEnrollment | empty | not }} + publicSubjectMap: {{ $target.subjectMap }} + cleanupRegistration: {{ $rid | quote }} + clientID: saw-portal-{{ $rid }} + keycloakNamespace: {{ $r.keycloakNamespace | quote }} + keycloakProvisionerSecret: {{ $r.keycloakProvisionerSecret | quote }} + subject: {{ (index ($site.subjects | default dict) $target.subjectMap | default $r.subject | default "") | quote }} + workspaceID: {{ default "" $r.workspaceID | quote }} + workspace: {{ $target.workspace }} + sandbox: {{ $target.sandbox }} + trustedNamespace: {{ $ctl }} + gatewayNamespace: {{ $ns }} + gateway: {{ $u }} + gatewayEndpoint: https://{{ $u }}-gateway.{{ $ns }}.svc:17670 + gatewayCA: {{ $r.gatewayCA | quote }} + issuer: {{ $r.issuer | quote }} + ownerLogin: {{ $u }} + ownerSubject: {{ $.Values.ownerSubject }} + installationID: {{ $.Values.installationID }} + host: {{ $u }}-{{ $target.workspace }}-{{ $target.sandbox }}-ui.apps.{{ $.Values.global.clusterDomain }} + sdkImage: {{ $e.image }} + imageBuildNamespace: {{ $e.buildNamespace }} + egress: {{ $r.egress | toJson }} + ignoreDifferences: + - {group: "", kind: Secret, name: ui-{{ $rid }}-registration-state, namespace: {{ $r.keycloakNamespace }}, jsonPointers: [/data]} + - {group: "", kind: Secret, name: ui-{{ $rid }}-browser-proof, namespace: {{ $ns }}, jsonPointers: [/data, /immutable]} + - {group: "", kind: Secret, name: relay-credentials, namespace: {{ $ui }}, jsonPointers: [/data]} + syncPolicy: + automated: {prune: true, selfHeal: true} + retry: {limit: 6, refresh: true} + syncOptions: [RespectIgnoreDifferences=true] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ ternary "relay-bind-dns" (printf "dns-%s" $rid) (eq (len $targets) 1) }}, namespace: {{ $ctl }}} +spec: + podSelector: {matchLabels: {app: ui-{{ $rid }}, role: bind}} + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: openshift-dns}} + podSelector: {matchLabels: {dns.operator.openshift.io/daemonset-dns: default}} + ports: [{protocol: UDP, port: 53}, {protocol: TCP, port: 53}, {protocol: UDP, port: 5353}, {protocol: TCP, port: 5353}] +{{- end }} +{{- end }}{{- end }} diff --git a/charts/saw-portal-container/templates/stack.yaml b/charts/saw-portal-container/templates/stack.yaml new file mode 100644 index 000000000..a0a540369 --- /dev/null +++ b/charts/saw-portal-container/templates/stack.yaml @@ -0,0 +1,154 @@ +{{- /* The site supplies reviewed values for the stock charts. No tpl: owner input + must never become a Helm program or choose a source, namespace or chart. */ -}} +{{- $site := .Values.site | default dict }} +{{- $stack := $site.stack | default dict }} +{{- if $stack }} +{{- $u := .Values.owner }} +{{- $reg := .Values.registration }} +{{- $runtime := .Release.Namespace }} +{{- $trusted := printf "saw-ctl-%s" $reg }} +{{- $archive := printf "saw-history-%s" $reg }} +{{- $g := .Values.global }} +{{- $ownerID := required "installationID required" .Values.installationID }} +{{- $ui := printf "saw-ui-%s" $reg }} +{{- $map := dict "RUNTIME" $runtime "TRUSTED" $trusted "ARCHIVE" $archive "UI" $ui "OWNER" $u "SUBJECT" .Values.ownerSubject "INSTALLATION" $ownerID "DOMAIN" $g.clusterDomain }} +{{- $targets := include "portal.targets" . | fromJson }} +{{- $namespaces := dict $archive "retained" }} +{{- range $slot := $targets }}{{- if $slot.ui }}{{- $_ := set $namespaces (printf "saw-ui-%s" $slot.relayID) "ephemeral" }}{{- end }}{{- end }} +{{- range $name, $dest := $namespaces }} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $name }} + labels: + {{- if eq $dest "ephemeral" }} + openshell.pattern/saw: "true" + {{- end }} + saw.redhat.com/runtime-owner: {{ $ownerID | quote }} + saw.redhat.com/registration: {{ $reg | quote }} + saw.redhat.com/portal-owner: {{ $u | quote }} + pod-security.kubernetes.io/enforce: restricted + {{- if eq $dest "retained" }} + annotations: {argocd.argoproj.io/compare-options: IgnoreExtraneous, argocd.argoproj.io/sync-options: 'Prune=false,Delete=false', argocd.argoproj.io/sync-wave: "-2"} + {{- else }} + annotations: {argocd.argoproj.io/sync-wave: "-2"} + {{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: default-deny, namespace: {{ $name }}, annotations: {argocd.argoproj.io/sync-wave: "-1"}} +spec: {podSelector: {}, policyTypes: [Ingress, Egress], ingress: [], egress: []} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: identity-api, namespace: {{ $trusted }}} +spec: + podSelector: {matchLabels: {openshell.pattern/role: identity-bootstrap}} + policyTypes: [Egress] + egress: {{ $stack.proxy.values.network.bootstrapEgress | toJson }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ printf "saw-%s-governance" $reg }}, namespace: {{ required "governance namespace required" $site.governanceNamespace }}} +spec: + podSelector: {matchLabels: {app.kubernetes.io/name: governance-interceptor}} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $runtime }}}} + podSelector: {matchLabels: {app.kubernetes.io/instance: {{ $u }}, openshell.pattern/role: gateway}} + ports: [{port: 18081, protocol: TCP}] +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $u }}-proxy-policy, namespace: {{ $trusted }}} +data: + config.json: | + {{ required "explicit governed provider policy required" $site.providerPolicy | toJson }} +{{- $charts := dict "identity" (list "saw-kubernetes-identity" $trusted) "policy" (list "saw-runtime-policies" $runtime) "proxy" (list "saw-kubernetes-proxy" $trusted) "gateway" (list "openshell-saw-kubernetes" $runtime) }} +{{- range $slot := $targets }}{{- $key := ternary "audit" (printf "audit-%s" $slot.relayID) (eq (len $targets) 1) }}{{- $_ := set $charts $key (list "saw-audit-writer" $archive $slot) }}{{- end }} +{{- range $key, $c := $charts }} +{{- $item := required (printf "missing site.stack.%s" $key) (get $stack (ternary "audit" $key (eq (len $c) 3))) }} +{{- if not (regexMatch "^[a-f0-9]{40}$" ($item.revision | default "")) }}{{ fail "each infrastructure chart must be pinned to a Git commit" }}{{ end }} +{{- $values := $item.values | toJson }} +{{- range $k, $v := $map }} +{{- $values = replace (printf "@%s@" $k) $v $values }} +{{- end }} +{{- if and (eq $key "gateway") $site.kataRuntimeClass }} +{{- $gateway := fromJson $values }} +{{- $_ := set $gateway "deployment" (dict "mode" "kata") }} +{{- $_ := set $gateway.kubernetes.workload "runtimeClassName" $site.kataRuntimeClass.name }} +{{- $values = toJson $gateway }} +{{- end }} +{{- if eq $key "policy" }} +{{- $policy := fromJson $values }} +{{- if $site.kataRuntimeClass }} +{{- $_ := set $policy "mode" "kata" }} +{{- $_ := set $policy "workloadRuntimeClassName" $site.kataRuntimeClass.name }} +{{- end }} +{{- $gw := $stack.gateway.values.kubernetes }} +{{- $slots := dict }} +{{- range $key, $slot := $targets }} +{{- $binding := dig "enrollment" "intent" "bindings" "bindings" $key dict $site }} +{{- $_ := set $slots (printf "%s--%s" $slot.workspace $slot.sandbox) (dict "image" ($slot.image | default "") "base" $slot.base "relayID" $slot.relayID "dataMounts" ($binding.mounts | default list)) }} +{{- end }} +{{- $_ := set $policy "slots" $slots }} +{{- $_ := set $policy "preparationImage" (dig "enrollment" "image" (dig "automaticEnrollment" "image" "" $site) $site) }} +{{- $_ := set $policy "forwarderImage" $stack.audit.values.image }} +{{- $_ := set $policy "serverTLSSecret" (replace "@OWNER@" $u $gw.identity.serverTlsSecret) }} +{{- $_ := set $policy "supervisorImage" (dig "supervisor" "image" $.Values.driverDefaults.supervisorImage $gw) }} +{{- $_ := set $policy "runtimeImage" ($gw.workload.runtimeImage | default $.Values.driverDefaults.runtimeImage) }} +{{- $proxyURL := replace "@TRUSTED@" $trusted (replace "@OWNER@" $u $gw.proxy.url) }} +{{- $_ := set $policy "supervisorCommand" (list "/openshell-supervisor" "--backend-descriptor-file" "/.openshell/supervisor/runtime-descriptor.json" "--auth-bundle-file" "/.openshell/supervisor/auth.json" "--workdir" "/sandbox" "--health-socket-path" "/run/openshell/health.sock" "--health-port" "5501" "--upstream-proxy" $proxyURL "--upstream-proxy-connect-by-hostname" "--upstream-proxy-ca-bundle" "/.openshell/supervisor/upstream-proxy-ca.pem") }} +{{- $values = toJson $policy }} +{{- end }} +{{- if and (eq $key "proxy") $site.providerPolicy }} +{{- $proxy := fromJson $values }} +{{- $secrets := list }} +{{- range $site.providerPolicy.bindings }}{{- $secrets = append $secrets .secret }}{{- end }} +{{- $_ := set $proxy "credentialSecretNames" ($secrets | uniq | sortAlpha) }} +{{- $_ := set $proxy "policyDigest" ($site.providerPolicy | toJson | sha256sum) }} +{{- $values = toJson $proxy }} +{{- end }} +{{- $release := $u }} +{{- if eq (len $c) 3 }} +{{- $slot := index $c 2 }} +{{- $release = $slot.base }} +{{- $v := $values | fromJson }} +{{- if and $site.lifecycleVersion $slot.modelInput }}{{- $_ := set $v "shareGateForModelSelection" true }}{{- end }} +{{- $_ := set $v.identity "workspace" $slot.workspace }}{{- $_ := set $v.identity "sandbox" $slot.sandbox }} +{{- $values = toJson $v }} +{{- end }} +{{- if regexMatch "@[A-Z_]+@" $values }}{{ fail "unknown site value placeholder" }}{{ end }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ printf "saw-c-%s-%s" $reg $key }} + namespace: {{ $g.vpArgoNamespace }} + labels: {saw.redhat.com/registration: {{ $reg | quote }}, openshell.pattern/owner: {{ $u | quote }}} + finalizers: [resources-finalizer.argocd.argoproj.io/foreground] +spec: + project: {{ $site.project | default "default" }} + destination: {server: 'https://kubernetes.default.svc', namespace: {{ index $c 1 }}} + source: + repoURL: {{ $g.repoURL | quote }} + targetRevision: {{ $item.revision | quote }} + path: charts/{{ index $c 0 }} + helm: + {{- if and (eq $key "gateway") (eq $stack.identity.revision $g.targetRevision) (eq $item.revision $g.targetRevision) }} + # The identity bootstrap independently checks both discovery endpoints + # against the destination API server before issuing gateway credentials. + apiVersions: + - agents.x-k8s.io/v1beta1/Sandbox + - policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy + {{- end }} + releaseName: {{ $release }} + valuesObject: {{ $values }} + syncPolicy: + automated: {prune: true, selfHeal: true} + retry: {limit: 8} +{{- end }} +{{- end }} diff --git a/charts/saw-portal-container/templates/status-reader.yaml b/charts/saw-portal-container/templates/status-reader.yaml new file mode 100644 index 000000000..02a332d84 --- /dev/null +++ b/charts/saw-portal-container/templates/status-reader.yaml @@ -0,0 +1,27 @@ +{{- $observer := (.Values.site | default dict).observer | default dict }} +{{- $portal := .Values.portalNamespace | default "saw-portal" }} +{{- $observerNamespace := $observer.namespace | default $portal }} +{{- $observerServiceAccount := $observer.serviceAccount | default "saw-workspaces-generator" }} +{{- $generatorServiceAccount := "saw-workspaces-generator" }} +{{- $provisionerServiceAccount := .Values.provisionerServiceAccount | default "saw-portal-provisioner" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: saw-portal-runtime-observer, namespace: {{ .Release.Namespace }}} +rules: + - apiGroups: [""] + resources: [configmaps] + resourceNames: [saw-bom-profiles, saw-runtime-intent, saw-runtime-status] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: saw-portal-runtime-observer, namespace: {{ .Release.Namespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: saw-portal-runtime-observer} +subjects: + - {kind: ServiceAccount, name: {{ $observerServiceAccount }}, namespace: {{ $observerNamespace }}} + {{- if or (ne $observerServiceAccount $generatorServiceAccount) (ne $observerNamespace $portal) }} + - {kind: ServiceAccount, name: {{ $generatorServiceAccount }}, namespace: {{ $portal }}} + {{- end }} + {{- if and (ne $provisionerServiceAccount $generatorServiceAccount) (or (ne $observerServiceAccount $provisionerServiceAccount) (ne $observerNamespace $portal)) }} + - {kind: ServiceAccount, name: {{ $provisionerServiceAccount }}, namespace: {{ $portal }}} + {{- end }} diff --git a/charts/saw-portal-container/values.yaml b/charts/saw-portal-container/values.yaml new file mode 100644 index 000000000..ec815deab --- /dev/null +++ b/charts/saw-portal-container/values.yaml @@ -0,0 +1,10 @@ +owner: "" +ownerSubject: "" +registration: "" +vaultPrefix: "" +secretStore: vault-backend + +driverDefaults: + gatewayVersion: 0.1.6-rhaiv.4 + supervisorImage: quay.io/opendatahub/odh-openshell-supervisor@sha256:626b91a0f53393dac0d591c142b81100916aa66b4e65444b5527672b5533fe86 + runtimeImage: quay.io/opendatahub/odh-openshell-sandbox@sha256:0cde11346e68485907c22d3c9b30802e52cf16e3853720a864ac8347bb68f6ef diff --git a/charts/saw-runtime-policies/Chart.yaml b/charts/saw-runtime-policies/Chart.yaml new file mode 100644 index 000000000..8dee57464 --- /dev/null +++ b/charts/saw-runtime-policies/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: saw-runtime-policies +description: Admission and OVN administrative network policies for stock OpenShell runtimes +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/charts/saw-runtime-policies/templates/_helpers.tpl b/charts/saw-runtime-policies/templates/_helpers.tpl new file mode 100644 index 000000000..93fdd4398 --- /dev/null +++ b/charts/saw-runtime-policies/templates/_helpers.tpl @@ -0,0 +1,46 @@ +{{- define "saw-runtime-policies.name" -}} +{{- printf "saw-%s-runtime" .Values.targetNamespace | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "saw-runtime-policies.validate" -}} +{{- if .Values.enabled -}} +{{- $ns := required "targetNamespace is required" .Values.targetNamespace -}} +{{- if or (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $ns)) (gt (len $ns) 50) (has $ns (list "default" "kube-system" "kube-public" "kube-node-lease")) (hasPrefix "openshift" $ns) (hasPrefix "kube-" $ns) -}} +{{- fail "targetNamespace must be an explicit non-system DNS label of at most 50 characters" -}} +{{- end -}} +{{- range $key := list "gatewayId" "gatewayServiceAccount" "sandboxServiceAccount" -}} +{{- $value := required (printf "%s is required" $key) (get $.Values $key) -}} +{{- if or (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $value)) (gt (len $value) 63) -}} +{{- fail (printf "%s must be a DNS label" $key) -}} +{{- end -}} +{{- end -}} +{{- if eq .Values.mode "kata" -}} +{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$" .Values.workloadRuntimeClassName) -}}{{- fail "kata requires an approved workload RuntimeClass name" -}}{{- end -}} +{{- else if or (ne .Values.mode "container") .Values.workloadRuntimeClassName -}}{{- fail "container placement must use the default runtime" -}}{{- end -}} +{{- if eq .Values.network.priority nil -}}{{- fail "network.priority must be explicitly allocated by the administrator" -}}{{- end -}} +{{- $priority := toString .Values.network.priority -}} +{{- if or (not (regexMatch "^[0-9]+$" $priority)) (gt (int $priority) 99) -}}{{- fail "network.priority must be an integer from 0 to 99" -}}{{- end -}} +{{- if not .Values.network.egress -}}{{- fail "network.egress must declare the required supervisor destinations" -}}{{- end -}} +{{- if not (.Capabilities.APIVersions.Has "policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy") -}}{{- fail "AdminNetworkPolicy API is required; install/reuse supported OVN-Kubernetes enforcement first" -}}{{- end -}} +{{- if not (.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy") -}}{{- fail "native ValidatingAdmissionPolicy API is required" -}}{{- end -}} +{{- $names := dict "deny-other-egress" true -}} +{{- range $rule := .Values.network.egress -}} +{{- if hasKey $names $rule.name -}}{{- fail "network.egress rule names must be unique and must not use deny-other-egress" -}}{{- end -}} +{{- $_ := set $names $rule.name true -}} +{{- if or (not $rule.to) (not $rule.ports) -}}{{- fail "each network.egress rule requires explicit to and ports" -}}{{- end -}} +{{- range $peer := $rule.to -}} +{{- if eq (len $peer) 0 -}}{{- fail "network.egress cannot contain an empty peer" -}}{{- end -}} +{{- if hasKey $peer "namespaces" -}} +{{- if and (empty $peer.namespaces.matchLabels) (empty $peer.namespaces.matchExpressions) -}}{{- fail "network.egress namespace selectors must be explicit" -}}{{- end -}} +{{- end -}} +{{- if hasKey $peer "pods" -}} +{{- if and (empty $peer.pods.namespaceSelector.matchLabels) (empty $peer.pods.namespaceSelector.matchExpressions) -}}{{- fail "network.egress pod peers need explicit namespace selectors" -}}{{- end -}} +{{- if and (empty $peer.pods.podSelector.matchLabels) (empty $peer.pods.podSelector.matchExpressions) -}}{{- fail "network.egress pod peers need explicit pod selectors" -}}{{- end -}} +{{- end -}} +{{- range $cidr := $peer.networks | default list -}} +{{- if has $cidr (list "0.0.0.0/0" "::/0") -}}{{- fail "network.egress cannot pass every destination; use approved explicit CIDRs" -}}{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/saw-runtime-policies/templates/admin-network-policy.yaml b/charts/saw-runtime-policies/templates/admin-network-policy.yaml new file mode 100644 index 000000000..48694cfc2 --- /dev/null +++ b/charts/saw-runtime-policies/templates/admin-network-policy.yaml @@ -0,0 +1,32 @@ +{{- include "saw-runtime-policies.validate" . -}} +{{- if .Values.enabled }} +apiVersion: policy.networking.k8s.io/v1alpha1 +kind: AdminNetworkPolicy +metadata: + name: {{ include "saw-runtime-policies.name" . }}-egress + annotations: + argocd.argoproj.io/sync-wave: "-10" + argocd.argoproj.io/sync-options: Prune=false +spec: + priority: {{ .Values.network.priority }} + subject: + pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.targetNamespace | quote }} + podSelector: + matchLabels: + openshell.ai/boundary-role: supervisor + openshell.ai/gateway-id: {{ .Values.gatewayId | quote }} + egress: +{{- range .Values.network.egress }} + - name: {{ .name | quote }} + action: Pass + to: {{ .to | toYaml | nindent 8 }} + ports: {{ .ports | toYaml | nindent 8 }} +{{- end }} + - name: deny-other-egress + action: Deny + to: + - networks: ["0.0.0.0/0", "::/0"] +{{- end }} diff --git a/charts/saw-runtime-policies/templates/kata-sandbox.yaml b/charts/saw-runtime-policies/templates/kata-sandbox.yaml new file mode 100644 index 000000000..254d60a4d --- /dev/null +++ b/charts/saw-runtime-policies/templates/kata-sandbox.yaml @@ -0,0 +1,37 @@ +{{- if and .Values.enabled (eq .Values.mode "kata") }} +{{- $name := printf "%s-kata" (include "saw-runtime-policies.name" .) }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ $name }} + annotations: + argocd.argoproj.io/sync-wave: "-10" + argocd.argoproj.io/sync-options: Prune=false +spec: + failurePolicy: Fail + matchConstraints: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.targetNamespace | quote }} + resourceRules: + - apiGroups: ["agents.x-k8s.io"] + apiVersions: ["v1beta1"] + operations: [CREATE, UPDATE] + resources: [sandboxes] + validations: + - expression: >- + has(object.spec.podTemplate.spec.runtimeClassName) && + object.spec.podTemplate.spec.runtimeClassName == {{ .Values.workloadRuntimeClassName | toJson }} + message: The Sandbox workload template must use the approved Kata RuntimeClass. +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ $name }} + annotations: + argocd.argoproj.io/sync-wave: "-9" + argocd.argoproj.io/sync-options: Prune=false +spec: + policyName: {{ $name }} + validationActions: [Deny] +{{- end }} diff --git a/charts/saw-runtime-policies/templates/runtime-validation.yaml b/charts/saw-runtime-policies/templates/runtime-validation.yaml new file mode 100644 index 000000000..baa4fb3ae --- /dev/null +++ b/charts/saw-runtime-policies/templates/runtime-validation.yaml @@ -0,0 +1,302 @@ +{{- if .Values.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ include "saw-runtime-policies.name" . }} + annotations: + argocd.argoproj.io/sync-wave: "-10" + argocd.argoproj.io/sync-options: Prune=false +spec: + failurePolicy: Fail + matchConstraints: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.targetNamespace | quote }} + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE, UPDATE] + resources: [pods] + variables: + - name: role + expression: >- + has(object.metadata.labels) && 'openshell.ai/boundary-role' in object.metadata.labels + ? object.metadata.labels['openshell.ai/boundary-role'] : '' + - name: sandboxAccount + expression: >- + has(object.spec.serviceAccountName) && object.spec.serviceAccountName == {{ .Values.sandboxServiceAccount | toJson }} + - name: previousRole + expression: >- + oldObject != null && has(oldObject.metadata.labels) && 'openshell.ai/boundary-role' in oldObject.metadata.labels + ? oldObject.metadata.labels['openshell.ai/boundary-role'] : '' + - name: previousSandboxAccount + expression: >- + oldObject != null && has(oldObject.spec.serviceAccountName) && + oldObject.spec.serviceAccountName == {{ .Values.sandboxServiceAccount | toJson }} + - name: runtimePod + expression: >- + variables.sandboxAccount || variables.role in ['workload', 'supervisor'] || + variables.previousSandboxAccount || variables.previousRole in ['workload', 'supervisor'] + - name: gatewayClaim + expression: >- + (has(object.spec.serviceAccountName) && object.spec.serviceAccountName == {{ .Values.gatewayServiceAccount | toJson }}) || + (has(object.metadata.labels) && 'openshell.pattern/role' in object.metadata.labels && + object.metadata.labels['openshell.pattern/role'] == 'gateway') + - name: previousGatewayClaim + expression: >- + oldObject != null && ((has(oldObject.spec.serviceAccountName) && + oldObject.spec.serviceAccountName == {{ .Values.gatewayServiceAccount | toJson }}) || + (has(oldObject.metadata.labels) && 'openshell.pattern/role' in oldObject.metadata.labels && + oldObject.metadata.labels['openshell.pattern/role'] == 'gateway')) + - name: owners + expression: "has(object.metadata.ownerReferences) ? object.metadata.ownerReferences : []" + - name: podLabels + expression: "has(object.metadata.labels) ? object.metadata.labels : {}" + - name: sandboxOwner + expression: >- + variables.owners.size() == 1 && variables.owners[0].kind == 'Sandbox' && + variables.owners[0].apiVersion in ['agents.x-k8s.io/v1alpha1','agents.x-k8s.io/v1beta1'] + - name: slot + expression: "variables.sandboxOwner ? variables.owners[0].name : ''" + - name: pair + expression: >- + 'openshell.ai/boundary-pair' in variables.podLabels ? variables.podLabels['openshell.ai/boundary-pair'] : '' + - name: gatewayCaller + expression: request.userInfo.username == {{ printf "system:serviceaccount:%s:%s" .Values.targetNamespace .Values.gatewayServiceAccount | toJson }} + - name: preparation + expression: >- + object.spec.serviceAccountName == 'saw-gate-preparation' && variables.owners.size() == 1 && + variables.owners[0].apiVersion == 'batch/v1' && variables.owners[0].kind == 'Job' && + has(variables.owners[0].controller) && variables.owners[0].controller && + 'batch.kubernetes.io/job-name' in variables.podLabels && + variables.podLabels['batch.kubernetes.io/job-name'] == variables.owners[0].name && + variables.owners[0].name.matches('^(saw-gate-preparation|gate-[a-f0-9]{16})-') + - name: forwarder + expression: >- + object.spec.serviceAccountName == 'default' && variables.owners.size() == 1 && + variables.owners[0].apiVersion == 'apps/v1' && variables.owners[0].kind == 'ReplicaSet' && + has(variables.owners[0].controller) && variables.owners[0].controller && + 'saw-audit-forward' in variables.podLabels && 'pod-template-hash' in variables.podLabels && + variables.owners[0].name == variables.podLabels['saw-audit-forward'] + '-forward-' + variables.podLabels['pod-template-hash'] + validations: + - expression: >- + (variables.gatewayClaim && !variables.runtimePod && !variables.preparation && !variables.forwarder) || + (variables.runtimePod && !variables.gatewayClaim && !variables.preparation && !variables.forwarder) || + ((variables.preparation || variables.forwarder) && !variables.gatewayClaim && variables.role == '') + message: Only the registered gateway, sandbox, preparation Job and audit forwarder Pods are allowed. + - expression: >- + request.operation != 'CREATE' || !variables.preparation || + request.userInfo.username in ['system:serviceaccount:kube-system:job-controller','system:kube-controller-manager'] + message: Only the Job controller may create preparation Pods. + - expression: >- + request.operation != 'CREATE' || !variables.forwarder || + request.userInfo.username in ['system:serviceaccount:kube-system:replicaset-controller','system:kube-controller-manager'] + message: Only the ReplicaSet controller may create audit forwarders. + - expression: >- + !(variables.preparation || variables.forwarder) || + (has(object.spec.automountServiceAccountToken) && !object.spec.automountServiceAccountToken && + object.spec.containers.size() == 1 && + (!has(object.spec.initContainers) || object.spec.initContainers.size() == 0) && + object.spec.containers.all(c, !has(c.envFrom) && (!has(c.env) || c.env.all(e, !has(e.valueFrom))) && + !has(c.lifecycle) && !has(c.volumeDevices)) && + object.spec.volumes.all(v, has(v.configMap) || has(v.persistentVolumeClaim) || + (variables.preparation && v.name == 'proof' && has(v.secret) && + v.secret.secretName.matches('^ui-[a-f0-9]{16}-browser-proof$')))) + message: Helper Pods cannot project unrelated Secrets or access API credentials. + - expression: >- + !variables.preparation || + (object.spec.containers[0].name == 'prep' && object.spec.containers[0].image == {{ .Values.preparationImage | toJson }} && + object.spec.containers[0].command.size() >= 3 && object.spec.containers[0].command[0] == 'python3' && + object.spec.containers[0].command[1] == '-m' && object.spec.containers[0].command[2] == 'saw_kubernetes.prepare_gate') + message: Gate preparation requires the pinned publisher executable. + - expression: >- + !variables.preparation || false + {{- range $slot := .Values.slots }} + || (variables.owners[0].name.startsWith({{ ternary "saw-gate-preparation-" (printf "gate-%s-" ($slot.relayID | default "missing")) (eq (len $.Values.slots) 1) | toJson }}) && + object.spec.volumes.exists(v, v.name == 'gate') && + object.spec.volumes.all(v, + (v.name == 'gate' && has(v.persistentVolumeClaim) && v.persistentVolumeClaim.claimName == {{ printf "saw-gate-%s" $slot.base | toJson }}) || + (v.name == 'bom' && has(v.configMap) && v.configMap.name == 'saw-bom-profiles') || + (v.name == 'proof' && has(v.secret) && v.secret.secretName == {{ printf "ui-%s-browser-proof" ($slot.relayID | default "missing") | toJson }})) && + object.spec.containers[0].volumeMounts.all(m, !has(m.subPath) && !has(m.subPathExpr) && + ((m.name == 'gate' && m.mountPath == '/volume') || + (m.name == 'proof' && m.mountPath == '/proof' && m.readOnly) || + (m.name == 'bom' && m.mountPath == '/bom' && m.readOnly)))) + {{- end }} + message: Preparation inputs must belong to the registered slot. + - expression: >- + !variables.forwarder || + (object.spec.containers[0].name == 'forward' && object.spec.containers[0].image == {{ .Values.forwarderImage | toJson }} && + object.spec.containers[0].command == ['node','/code/main.mjs'] && + object.spec.volumes.all(v, (v.name == 'data' && has(v.persistentVolumeClaim) && + v.persistentVolumeClaim.claimName == 'saw-audit-' + variables.podLabels['saw-audit-forward']) || + (v.name in ['code','trust'] && has(v.configMap) && + v.configMap.name == variables.podLabels['saw-audit-forward'] + (v.name == 'code' ? '-forward' : '-ca')))) + message: Audit forwarders require their pinned script and slot-specific exchange. + - expression: >- + !variables.forwarder || false + {{- range $slot := .Values.slots }} + || variables.podLabels['saw-audit-forward'] == {{ $slot.base | toJson }} + {{- end }} + message: The audit forwarder must belong to a registered slot. + - expression: >- + !('saw-audit-forward' in variables.podLabels) || variables.forwarder + message: Audit forwarder network identity is reserved for its ReplicaSet. + - expression: >- + variables.role != 'workload' || (variables.sandboxOwner && has(variables.owners[0].controller) && + variables.owners[0].controller && variables.slot == object.metadata.name && + (request.operation != 'CREATE' || request.userInfo.username == {{ .Values.sandboxControllerUsername | toJson }})) + message: Only the Agent Sandbox controller may create its named workload Pod. + - expression: >- + request.operation != 'UPDATE' || + (oldObject.metadata.ownerReferences == object.metadata.ownerReferences && + oldObject.spec.containers == object.spec.containers && + oldObject.spec.?initContainers == object.spec.?initContainers && + oldObject.spec.?volumes == object.spec.?volumes && + (!variables.gatewayCaller || (variables.runtimePod && oldObject.metadata.labels == object.metadata.labels && + oldObject.metadata.?annotations == object.metadata.?annotations))) + message: Workspace Pod images, containers, volumes and ownership are immutable. + - expression: >- + !variables.runtimePod || (variables.sandboxOwner && variables.pair.matches('^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$') && + has(object.spec.automountServiceAccountToken) && !object.spec.automountServiceAccountToken && + object.spec.containers.size() == 1 && (!has(object.spec.ephemeralContainers) || object.spec.ephemeralContainers.size() == 0)) + message: Runtime Pods require one stock container and an explicit token boundary. + - expression: >- + !variables.runtimePod || (variables.slot in {{ keys .Values.slots | sortAlpha | toJson }} && + object.spec.containers.all(c, !has(c.envFrom) && (!has(c.env) || c.env.all(e, !has(e.valueFrom))) && + !has(c.lifecycle) && !has(c.volumeDevices) && !has(c.args) && !has(c.startupProbe) && + !has(c.livenessProbe) && (!has(c.readinessProbe) || has(c.readinessProbe.tcpSocket)))) + message: Sandbox containers must use the approved slot and stock execution contract. + - expression: >- + variables.role != 'supervisor' || (!variables.owners[0].?controller.orValue(false) && object.metadata.name == 'os-supervisor-' + variables.pair && + (!has(object.spec.initContainers) || object.spec.initContainers.size() == 0) && + object.spec.containers[0].name == 'supervisor' && + object.spec.containers[0].image == {{ .Values.supervisorImage | toJson }} && + object.spec.containers[0].command == {{ .Values.supervisorCommand | toJson }} && + object.spec.containers[0].env.all(e, e.name in ['OPENSHELL_ADMITTED_ISOLATION_BACKEND','OPENSHELL_ENDPOINT', + 'OPENSHELL_SANDBOX_ID','OPENSHELL_SANDBOX','OPENSHELL_MAIN_PROCESS_SPEC','OPENSHELL_K8S_SA_TOKEN_FILE', + 'OPENSHELL_SSH_SOCKET_PATH','OPENSHELL_SSH_SOCKET_SHARED','OPENSHELL_PROXY_TLS_DIR','OPENSHELL_PROXY_CA_CERT', + 'OPENSHELL_PROXY_CA_KEY','OPENSHELL_LOG_LEVEL','OPENSHELL_TELEMETRY_ENABLED','OPENSHELL_NETWORK_RUNTIME_CAPABILITIES', + 'OPENSHELL_TLS_CA','OPENSHELL_GATEWAY_TLS_SERVER_NAME','OPENSHELL_GATEWAY_NAME']) && + object.spec.volumes.all(v, has(v.emptyDir) || + (v.name == 'bootstrap' && has(v.secret) && v.secret.secretName.matches('^os-supervisor-' + variables.pair + '-[a-f0-9]{12}$')) || + (v.name == 'client-tls' && has(v.secret) && v.secret.secretName == {{ .Values.serverTLSSecret | toJson }} && + has(v.secret.items) && v.secret.items.size() == 1 && v.secret.items[0].key == 'ca.crt' && v.secret.items[0].path == 'ca.crt') || + (v.name == 'sa-token' && has(v.projected) && v.projected.sources.size() == 1 && + has(v.projected.sources[0].serviceAccountToken) && v.projected.sources[0].serviceAccountToken.audience == 'openshell-gateway' && + v.projected.sources[0].serviceAccountToken.path == 'token' && v.projected.sources[0].serviceAccountToken.expirationSeconds <= 3600))) + message: Supervisor executable, TLS projection and token must match the stock contract. + - expression: >- + variables.role != 'workload' || (object.spec.containers[0].name == 'agent' && + (!has(object.spec.containers[0].env) || object.spec.containers[0].env.all(e, e.name in ['TMPDIR', + 'OPENSHELL_TELEMETRY_ENABLED','OPENSHELL_OCI_IMAGE_USER','OPENSHELL_SANDBOX_UID','OPENSHELL_SANDBOX_GID'])) && + object.spec.containers[0].command == ['/.openshell/runtime/openshell-sandbox','--bootstrap','/.openshell/state/bootstrap/boundary.json'] && + has(object.spec.initContainers) && object.spec.initContainers.size() == 1 && + object.spec.initContainers.all(c, c.image == {{ .Values.runtimeImage | toJson }} && c.name == 'openshell-sandbox-bootstrap' && + c.command == ['/openshell-sandbox','bootstrap'] && !has(c.args) && !has(c.env) && !has(c.envFrom) && + !has(c.lifecycle) && !has(c.volumeDevices) && !has(c.startupProbe) && !has(c.livenessProbe) && !has(c.readinessProbe))) + message: Workloads must use the stock runtime bootstrap. +{{- range $name, $slot := .Values.slots }} + - expression: >- + variables.role != 'workload' || variables.slot != {{ $name | toJson }} || ( + object.spec.containers[0].image == {{ $slot.image | toJson }} && object.spec.containers[0].image.matches("@sha256:[a-f0-9]{64}$") && + object.spec.volumes.all(v, (has(v.emptyDir) && v.name in ['openshell-runtime','openshell-runtime-state','openshell-run']) || (v.name == 'openshell-pod-identity' && has(v.downwardAPI)) || + (v.name == 'openshell-sandbox-bootstrap' && has(v.secret) && v.secret.secretName.matches('^os-sandbox-' + variables.pair + '-[a-f0-9]{12}$')) || + (has(v.persistentVolumeClaim) && ( + {{ range $i, $m := $slot.dataMounts -}} + (v.name == 'data-{{ $i }}' && v.persistentVolumeClaim.claimName == {{ $m.claimName | toJson }} && + v.persistentVolumeClaim.?readOnly.orValue(false) == {{ $m.readOnly }}) || + {{ end -}} + (v.name == 'saw-workspace' && v.persistentVolumeClaim.claimName == {{ printf "workspace-%s" $slot.base | toJson }}) || + (v.name == 'saw-gate' && v.persistentVolumeClaim.claimName == {{ printf "saw-gate-%s" $slot.base | toJson }} && v.persistentVolumeClaim.readOnly) || + (v.name == 'saw-audit' && v.persistentVolumeClaim.claimName == {{ printf "saw-audit-%s" $slot.base | toJson }})))) && + object.spec.containers[0].volumeMounts.all(m, !has(m.subPathExpr) && ( + {{ range $i, $m := $slot.dataMounts -}} + (m.name == 'data-{{ $i }}' && m.mountPath == {{ $m.mountPath | toJson }} && + !has(m.subPath) && m.?readOnly.orValue(false) == {{ $m.readOnly }}) || + {{ end -}} + (m.name == 'saw-workspace' && ((m.mountPath == '/sandbox/workspace' && !has(m.subPath)) || (m.mountPath == '/sandbox/briefing' && m.subPath == 'briefing'))) || + (m.name == 'saw-audit' && m.subPath == 'log' && m.mountPath == '/sandbox/.saw/audit') || + (m.name == 'openshell-runtime' && m.mountPath == '/.openshell/runtime' && m.readOnly) || + (m.name == 'openshell-runtime-state' && m.mountPath == '/.openshell/state') || + (m.name == 'openshell-pod-identity' && m.mountPath == '/.openshell/pod-identity' && m.readOnly) || + (m.name == 'openshell-run' && m.mountPath == '/run') || m.name == 'saw-gate')) && + object.spec.containers[0].volumeMounts.all(m, + m.name != 'saw-gate' || (m.readOnly && has(m.subPath) && ((m.mountPath == '/sandbox/.saw/trusted' && m.subPath == 'bundle') || (m.mountPath == '/sandbox/harness' && m.subPath.matches('^harness-[a-f0-9]{64}$'))))) && + object.spec.initContainers[0].volumeMounts.all(m, m.name in ['openshell-sandbox-bootstrap','openshell-runtime','openshell-runtime-state'])) + message: Workload image and PVC attachments must match the approved sandbox slot. +{{- end }} + - expression: >- + !variables.gatewayClaim || (has(object.spec.serviceAccountName) && + object.spec.serviceAccountName == {{ .Values.gatewayServiceAccount | toJson }} && + has(object.metadata.labels) && 'openshell.pattern/role' in object.metadata.labels && + object.metadata.labels['openshell.pattern/role'] == 'gateway' && + 'app.kubernetes.io/instance' in object.metadata.labels && + object.metadata.labels['app.kubernetes.io/instance'] == {{ .Values.gatewayId | toJson }} && + has(object.metadata.ownerReferences) && object.metadata.ownerReferences.exists(r, + r.kind == 'StatefulSet' && r.apiVersion == 'apps/v1' && has(r.controller) && r.controller && + r.name == {{ printf "%s-gateway" .Values.gatewayId | toJson }})) + message: Gateway identity requires the approved ServiceAccount, instance and controlling StatefulSet. + reason: Forbidden + - expression: >- + request.operation != 'UPDATE' || variables.previousGatewayClaim == variables.gatewayClaim + message: Pods cannot acquire or drop gateway identity after creation. + reason: Forbidden + - expression: >- + request.operation != 'CREATE' || !variables.gatewayClaim || request.userInfo.username in + ['system:serviceaccount:kube-system:statefulset-controller', 'system:kube-controller-manager'] + message: Only the Kubernetes StatefulSet controller may create a gateway Pod. + reason: Forbidden + - expression: >- + !variables.runtimePod || (variables.sandboxAccount && variables.role in ['workload', 'supervisor']) + message: OpenShell runtime Pods must preserve the approved ServiceAccount and role labels. + reason: Forbidden + - expression: >- + !variables.runtimePod || variables.previousRole == '' || variables.previousRole == variables.role + message: OpenShell runtime Pod role labels cannot change after creation. + reason: Forbidden + - expression: >- + !variables.runtimePod || (has(object.metadata.ownerReferences) && object.metadata.ownerReferences.exists(r, + r.kind == 'Sandbox' && r.apiVersion in ['agents.x-k8s.io/v1alpha1', 'agents.x-k8s.io/v1beta1'])) + message: OpenShell runtime Pods must retain their Sandbox owner reference. + reason: Forbidden + - expression: >- + variables.role != 'supervisor' || (has(object.metadata.labels) && + 'openshell.ai/gateway-id' in object.metadata.labels && + object.metadata.labels['openshell.ai/gateway-id'] == {{ .Values.gatewayId | toJson }}) + message: Supervisor Pods must retain their approved gateway identity. + reason: Forbidden + - expression: >- + request.operation != 'CREATE' || variables.role != 'supervisor' || + request.userInfo.username == {{ printf "system:serviceaccount:%s:%s" .Values.targetNamespace .Values.gatewayServiceAccount | toJson }} + message: Only this workspace gateway may create supervisor Pods. + reason: Forbidden + - expression: >- + variables.role != 'supervisor' || !has(object.spec.runtimeClassName) || object.spec.runtimeClassName == '' + message: Stock supervisor Pods must use the default container runtime. + reason: Forbidden + - expression: >- +{{- if eq .Values.mode "kata" }} + variables.role != 'workload' || (has(object.spec.runtimeClassName) && object.spec.runtimeClassName == {{ .Values.workloadRuntimeClassName | toJson }}) + message: Kata workloads require the approved RuntimeClass. +{{- else }} + variables.role != 'workload' || !has(object.spec.runtimeClassName) || object.spec.runtimeClassName == '' + message: Container-mode workload Pods must use the default container runtime. +{{- end }} + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ include "saw-runtime-policies.name" . }} + annotations: + argocd.argoproj.io/sync-wave: "-9" + argocd.argoproj.io/sync-options: Prune=false +spec: + policyName: {{ include "saw-runtime-policies.name" . }} + validationActions: [Deny] + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.targetNamespace | quote }} +{{- end }} diff --git a/charts/saw-runtime-policies/tests/container-values.yaml b/charts/saw-runtime-policies/tests/container-values.yaml new file mode 100644 index 000000000..e01133e75 --- /dev/null +++ b/charts/saw-runtime-policies/tests/container-values.yaml @@ -0,0 +1,60 @@ +# Render-only fixture. Replace destinations and priority before cluster use. +enabled: true +targetNamespace: saw-alice +gatewayId: saw-alice-gateway +gatewayServiceAccount: saw-alice-gateway +sandboxServiceAccount: saw-sandbox +mode: container +network: + priority: 70 + egress: + - name: dns + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + podSelector: + matchLabels: + dns.operator.openshift.io/daemonset-dns: default + ports: + - portNumber: {protocol: UDP, port: 5353} + - portNumber: {protocol: TCP, port: 5353} + - name: gateway + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: saw-alice + podSelector: + matchLabels: + app.kubernetes.io/component: gateway + ports: + - portNumber: {protocol: TCP, port: 8080} + - name: workload-boundary + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: saw-alice + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + ports: + - portNumber: {protocol: TCP, port: 5500} + - name: credential-proxy + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: saw-alice + podSelector: + matchLabels: + app.kubernetes.io/component: credential-proxy + ports: + - portNumber: {protocol: TCP, port: 3128} + - name: approved-endpoint + to: + - networks: ["192.0.2.10/32", "2001:db8::10/128"] + ports: + - portNumber: {protocol: TCP, port: 443} diff --git a/charts/saw-runtime-policies/values.schema.json b/charts/saw-runtime-policies/values.schema.json new file mode 100644 index 000000000..a8834a079 --- /dev/null +++ b/charts/saw-runtime-policies/values.schema.json @@ -0,0 +1,307 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "targetNamespace": { + "type": "string" + }, + "gatewayId": { + "type": "string" + }, + "gatewayServiceAccount": { + "type": "string" + }, + "sandboxServiceAccount": { + "type": "string" + }, + "mode": { + "enum": [ + "container", + "kata" + ] + }, + "workloadRuntimeClassName": {"type": "string"}, + "network": { + "type": "object", + "additionalProperties": false, + "properties": { + "priority": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 99 + }, + "egress": { + "type": "array", + "maxItems": 99, + "items": { + "$ref": "#/definitions/egressRule" + } + } + }, + "required": [ + "egress" + ] + }, + "global": { + "type": "object" + }, + "sandboxControllerUsername": { + "type": "string" + }, + "supervisorImage": { + "type": "string" + }, + "runtimeImage": { + "type": "string" + }, + "serverTLSSecret": { + "type": "string" + }, + "slots": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "base": { + "type": "string" + }, + "relayID": { + "type": "string", + "pattern": "^[a-f0-9]{16}$" + }, + "dataMounts": { + "type": "array", + "maxItems": 16, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "claimName", + "mountPath", + "uid", + "readOnly" + ], + "properties": { + "claimName": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9.-]*$", + "not": { + "pattern": "^saw-(gate|audit)-" + } + }, + "mountPath": { + "type": "string", + "pattern": "^/data(/[a-zA-Z0-9._-]+)*$", + "not": { + "pattern": "/\\.{1,2}(/|$)" + } + }, + "uid": { + "type": "string", + "minLength": 1 + }, + "readOnly": { + "type": "boolean" + } + } + } + } + }, + "required": [ + "image", + "base" + ], + "additionalProperties": false + } + }, + "supervisorCommand": { + "type": "array", + "items": { + "type": "string" + } + }, + "preparationImage": { + "type": "string" + }, + "forwarderImage": { + "type": "string" + } + }, + "required": [ + "enabled", + "targetNamespace", + "gatewayId", + "gatewayServiceAccount", + "sandboxServiceAccount", + "mode", + "network" + ], + "definitions": { + "selector": { + "type": "object", + "additionalProperties": false, + "properties": { + "matchLabels": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "matchExpressions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "operator" + ], + "properties": { + "key": { + "type": "string", + "minLength": 1 + }, + "operator": { + "enum": [ + "In", + "NotIn", + "Exists", + "DoesNotExist" + ] + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "anyOf": [ + { + "required": [ + "matchLabels" + ] + }, + { + "required": [ + "matchExpressions" + ] + } + ] + }, + "peer": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "namespaces": { + "$ref": "#/definitions/selector" + }, + "pods": { + "type": "object", + "additionalProperties": false, + "required": [ + "namespaceSelector", + "podSelector" + ], + "properties": { + "namespaceSelector": { + "$ref": "#/definitions/selector" + }, + "podSelector": { + "$ref": "#/definitions/selector" + } + } + }, + "networks": { + "type": "array", + "minItems": 1, + "maxItems": 25, + "items": { + "type": "string", + "minLength": 3 + } + } + } + }, + "port": { + "type": "object", + "additionalProperties": false, + "required": [ + "portNumber" + ], + "properties": { + "portNumber": { + "type": "object", + "additionalProperties": false, + "required": [ + "protocol", + "port" + ], + "properties": { + "protocol": { + "enum": [ + "TCP", + "UDP", + "SCTP" + ] + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + } + } + } + } + }, + "egressRule": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "to", + "ports" + ], + "properties": { + "name": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + }, + "to": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { + "$ref": "#/definitions/peer" + } + }, + "ports": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { + "$ref": "#/definitions/port" + } + } + } + } + }, + "additionalProperties": false +} diff --git a/charts/saw-runtime-policies/values.yaml b/charts/saw-runtime-policies/values.yaml new file mode 100644 index 000000000..6ee07a152 --- /dev/null +++ b/charts/saw-runtime-policies/values.yaml @@ -0,0 +1,25 @@ +# Platform-owned policy release; disabled until dependencies and site inputs exist. +enabled: false +targetNamespace: "" +gatewayId: "" +gatewayServiceAccount: "" +sandboxServiceAccount: "" +mode: container +workloadRuntimeClassName: "" +network: + # Required administrator allocation, unique among overlapping ANPs (0..99). + priority: null + # ANP egress peers/ports. The chart fixes each action to Pass, then denies + # all remaining IPv4/IPv6. Populate actual DNS, gateway, workload-boundary, + # credential-proxy and approved external destinations before enabling. + egress: [] + +# Filled by the shared composition from the approved BOM and gateway configuration. +sandboxControllerUsername: system:serviceaccount:agent-sandbox-system:agent-sandbox-controller +slots: {} +supervisorImage: "" +runtimeImage: "" +serverTLSSecret: "" +supervisorCommand: [] +preparationImage: "" +forwarderImage: "" diff --git a/charts/saw-users/files/governed-profiles.json b/charts/saw-users/files/governed-profiles.json new file mode 100644 index 000000000..6819eb497 --- /dev/null +++ b/charts/saw-users/files/governed-profiles.json @@ -0,0 +1,550 @@ +{ + "anthropic": { + "binaries": [ + "/usr/bin/curl", + "/usr/local/bin/curl", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "inference", + "credentials": [ + { + "auth_style": "header", + "description": "Anthropic API key", + "env_vars": [ + "ANTHROPIC_API_KEY" + ], + "header_name": "x-api-key", + "name": "api_key", + "required": true + } + ], + "description": "Anthropic inference API", + "discovery": { + "credentials": [ + "api_key" + ] + }, + "display_name": "Anthropic", + "endpoints": [ + { + "access": "read-write", + "enforcement": "enforce", + "host": "api.anthropic.com", + "port": 443, + "protocol": "rest" + } + ], + "id": "anthropic", + "inference_capable": true + }, + "brave": { + "binaries": [ + "/usr/bin/curl", + "/usr/local/bin/curl", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "other", + "credentials": [ + { + "auth_style": "header", + "description": "Brave Search API key", + "env_vars": [ + "BRAVE_API_KEY" + ], + "header_name": "X-Subscription-Token", + "name": "api_key", + "required": true + } + ], + "description": "Brave Search API access for governed sandbox agents", + "discovery": { + "credentials": [ + "api_key" + ] + }, + "display_name": "Brave Search", + "endpoints": [ + { + "access": "read-only", + "enforcement": "enforce", + "host": "api.search.brave.com", + "port": 443, + "protocol": "rest" + } + ], + "id": "brave" + }, + "gemini": { + "binaries": [ + "/usr/bin/gemini", + "/usr/lib/node_modules/@google/gemini-cli/dist/index.js", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "other", + "credentials": [ + { + "auth_style": "query", + "description": "Google Gemini API key", + "env_vars": [ + "GEMINI_API_KEY" + ], + "name": "api_key", + "query_param": "key", + "required": true + } + ], + "description": "Google Gemini AI and Vertex AI API access", + "discovery": { + "credentials": [ + "api_key" + ] + }, + "display_name": "Google Gemini", + "endpoints": [ + { + "access": "read-write", + "enforcement": "enforce", + "host": "generativelanguage.googleapis.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-write", + "enforcement": "enforce", + "host": "*-aiplatform.googleapis.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-write", + "enforcement": "enforce", + "host": "cloudcode-pa.googleapis.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "accounts.google.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "oauth2.googleapis.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "www.googleapis.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "iamcredentials.googleapis.com", + "port": 443, + "protocol": "rest" + } + ], + "id": "gemini" + }, + "github": { + "binaries": [ + "/usr/bin/gh", + "/usr/local/bin/gh", + "/usr/bin/git", + "/usr/local/bin/git", + "/usr/bin/curl", + "/usr/local/bin/curl" + ], + "category": "source_control", + "credentials": [ + { + "auth_style": "bearer", + "description": "GitHub token", + "env_vars": [ + "GITHUB_TOKEN", + "GH_TOKEN" + ], + "header_name": "authorization", + "name": "api_token", + "required": true + } + ], + "description": "GitHub API and Git operations", + "discovery": { + "credentials": [ + "api_token" + ] + }, + "display_name": "GitHub", + "endpoints": [ + { + "access": "read-only", + "enforcement": "enforce", + "host": "api-1.github.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "api.github.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "api.github.com", + "path": "/graphql", + "port": 443, + "protocol": "graphql" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "github.com", + "port": 443, + "protocol": "rest" + } + ], + "id": "github" + }, + "gmail": { + "binaries": [ + "/usr/bin/curl", + "/usr/local/bin/curl", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "messaging", + "credentials": [ + { + "auth_style": "bearer", + "description": "Gmail OAuth access token (gmail.readonly)", + "env_vars": [ + "GMAIL_ACCESS_TOKEN" + ], + "header_name": "authorization", + "name": "access_token", + "refresh": { + "material": [ + { + "description": "Google OAuth client ID", + "name": "client_id", + "required": true + }, + { + "description": "Google OAuth client secret", + "name": "client_secret", + "required": true, + "secret": true + }, + { + "description": "Google OAuth refresh token (scope gmail.readonly)", + "name": "refresh_token", + "required": true, + "secret": true + } + ], + "max_lifetime_seconds": 3600, + "refresh_before_seconds": 300, + "strategy": "oauth2_refresh_token", + "token_url": "https://oauth2.googleapis.com/token" + }, + "required": true + } + ], + "description": "Read-only Gmail API access for governed sandbox agents", + "discovery": { + "credentials": [ + "access_token" + ] + }, + "display_name": "Gmail", + "endpoints": [ + { + "enforcement": "enforce", + "host": "gmail.googleapis.com", + "port": 443, + "protocol": "rest", + "rules": [ + { + "allow": { + "method": "GET", + "path": "/gmail/v1/users/me/profile" + } + }, + { + "allow": { + "method": "GET", + "path": "/gmail/v1/users/me/labels" + } + }, + { + "allow": { + "method": "GET", + "path": "/gmail/v1/users/me/messages" + } + }, + { + "allow": { + "method": "GET", + "path": "/gmail/v1/users/me/messages/*" + } + }, + { + "allow": { + "method": "GET", + "path": "/gmail/v1/users/me/threads" + } + }, + { + "allow": { + "method": "GET", + "path": "/gmail/v1/users/me/threads/*" + } + } + ] + } + ], + "id": "gmail" + }, + "nvidia": { + "binaries": [ + "/usr/bin/curl", + "/usr/local/bin/curl", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "inference", + "credentials": [ + { + "auth_style": "bearer", + "description": "NVIDIA API key", + "env_vars": [ + "NVIDIA_API_KEY" + ], + "header_name": "authorization", + "name": "api_key", + "required": true + } + ], + "description": "NVIDIA API inference endpoints", + "discovery": { + "credentials": [ + "api_key" + ] + }, + "display_name": "NVIDIA Endpoints", + "endpoints": [ + { + "access": "read-write", + "enforcement": "enforce", + "host": "integrate.api.nvidia.com", + "port": 443, + "protocol": "rest" + } + ], + "id": "nvidia", + "inference_capable": true + }, + "openai": { + "binaries": [ + "/usr/bin/curl", + "/usr/local/bin/curl", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "inference", + "credentials": [ + { + "auth_style": "bearer", + "description": "OpenAI API key", + "env_vars": [ + "OPENAI_API_KEY" + ], + "header_name": "authorization", + "name": "api_key", + "required": true + } + ], + "description": "OpenAI inference API", + "discovery": { + "credentials": [ + "api_key" + ] + }, + "display_name": "OpenAI", + "endpoints": [ + { + "access": "read-write", + "enforcement": "enforce", + "host": "api.openai.com", + "port": 443, + "protocol": "rest" + } + ], + "id": "openai", + "inference_capable": true + }, + "slack": { + "binaries": [ + "/usr/bin/curl", + "/usr/local/bin/curl", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "messaging", + "credentials": [ + { + "auth_style": "bearer", + "description": "Slack bot or user token", + "env_vars": [ + "SLACK_BOT_TOKEN", + "SLACK_TOKEN" + ], + "header_name": "authorization", + "name": "api_token", + "refresh": { + "material": [ + { + "description": "Slack app client ID", + "name": "client_id", + "required": true + }, + { + "description": "Slack app client secret", + "name": "client_secret", + "required": true, + "secret": true + }, + { + "description": "Slack refresh token (xoxe-...)", + "name": "refresh_token", + "required": true, + "secret": true + } + ], + "max_lifetime_seconds": 43200, + "refresh_before_seconds": 600, + "strategy": "oauth2_refresh_token", + "token_url": "https://slack.com/api/oauth.v2.access" + }, + "required": true + } + ], + "description": "Read-only Slack Web API access for governed sandbox agents", + "discovery": { + "credentials": [ + "api_token" + ] + }, + "display_name": "Slack", + "endpoints": [ + { + "enforcement": "enforce", + "host": "slack.com", + "port": 443, + "protocol": "rest", + "rules": [ + { + "allow": { + "method": "GET", + "path": "/api/auth.test" + } + }, + { + "allow": { + "method": "GET", + "path": "/api/team.info" + } + }, + { + "allow": { + "method": "GET", + "path": "/api/users.info" + } + }, + { + "allow": { + "method": "GET", + "path": "/api/conversations.list" + } + }, + { + "allow": { + "method": "GET", + "path": "/api/conversations.info" + } + }, + { + "allow": { + "method": "GET", + "path": "/api/conversations.history" + } + }, + { + "allow": { + "method": "GET", + "path": "/api/conversations.replies" + } + } + ] + } + ], + "id": "slack" + }, + "web-search": { + "binaries": [ + "/usr/bin/curl", + "/usr/local/bin/curl", + "/usr/bin/node", + "/usr/bin/node-*", + "/usr/local/bin/node" + ], + "category": "other", + "description": "Tavily and Brave Search API endpoints and plugin registry", + "display_name": "Web Search", + "endpoints": [ + { + "access": "read-only", + "enforcement": "enforce", + "host": "api.tavily.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "api.search.brave.com", + "port": 443, + "protocol": "rest" + }, + { + "access": "read-only", + "enforcement": "enforce", + "host": "registry.npmjs.org", + "port": 443, + "protocol": "rest" + } + ], + "id": "web-search" + } +} diff --git a/charts/saw-users/files/preflight.py b/charts/saw-users/files/preflight.py new file mode 100644 index 000000000..c32862180 --- /dev/null +++ b/charts/saw-users/files/preflight.py @@ -0,0 +1,477 @@ +"""Shared, read-only checks for SAW Kubernetes workspace provisioning. + +Callers collect facts from their cluster client. This module validates only +public configuration and observed resource metadata. It never reads Secrets. +""" + +import hashlib +import json +import re + + +class PreflightError(ValueError): + """A failed check with a fixed, safe status code.""" + + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +CHECK_REASONS = { + "api": "ApiUnavailable", + "version": "ClusterVersionUnavailable", + "osc": "SandboxedContainersUnavailable", + "kataConfig": "KataConfigNotReady", + "workers": "KataWorkersNotReady", + "runtimeClass": "RuntimeClassRejected", + "agentSandbox": "AgentSandboxUnavailable", + "admission": "AdmissionUnavailable", + "network": "NetworkPolicyRejected", + "argo": "ArgoAccessDenied", + "storage": "StorageUnavailable", + "secrets": "SecretServicesUnavailable", + "registry": "RegistryUnavailable", + "providerEgress": "ProviderEgressRejected", + "revisions": "UnpinnedRevision", +} + +COMMON_CHECKS = ( + "api", "version", "agentSandbox", "admission", "network", "argo", + "storage", "secrets", "registry", "providerEgress", "revisions", +) +KATA_CHECKS = ("osc", "kataConfig", "workers", "runtimeClass") + +# This pinned image has /sandbox mode 0770, owned by 1000:0. The OpenShift +# namespace UID cannot traverse it. The reviewed notebook build adds o+x. +NON_TRAVERSABLE_WORKLOAD_IMAGES = frozenset({ + "quay.io/aipcc/base-images/agentic/openclaw@sha256:" + "896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", +}) + + +def verify_profile_image_access(site, profile_name, catalog): + """Reject known inaccessible workload images before enrollment starts.""" + profile = catalog.get(profile_name) if isinstance(catalog, dict) else None + if not isinstance(profile, dict) or not isinstance(site, dict): + raise PreflightError("WorkloadImageApprovalMissing") + overrides = site.get("profileImageOverrides", {}) + if not isinstance(overrides, dict): + raise PreflightError("WorkloadImageApprovalInvalid") + selected = overrides.get(profile_name, {}) + if not isinstance(selected, dict): + raise PreflightError("WorkloadImageApprovalInvalid") + for workspace in profile.get("workspaces", []): + if not isinstance(workspace, dict) or not workspace.get("enabled"): + continue + for sandbox in workspace.get("sandboxes", []): + if not isinstance(sandbox, dict) or not sandbox.get("enabled"): + continue + image = sandbox.get("image") + if image in NON_TRAVERSABLE_WORKLOAD_IMAGES: + replacement = selected.get(sandbox.get("name"), image) + if replacement == image: + raise PreflightError("WorkloadImageNotNamespaceAccessible") + if (not isinstance(replacement, str) or not re.fullmatch( + r"[a-z0-9][a-z0-9./:_-]+@sha256:[a-f0-9]{64}", replacement)): + raise PreflightError("WorkloadImageApprovalInvalid") + return True + + +def runtime_placement(runtime_class): + """Return the canonical placement payload used by owner approvals.""" + return {key: runtime_class.get(key) for key in ("handler", "overhead", "scheduling")} + + +def runtime_placement_digest(runtime_class): + raw = json.dumps(runtime_placement(runtime_class), sort_keys=True, + separators=(",", ":"), ensure_ascii=True).encode() + return hashlib.sha256(raw).hexdigest() + + +def fingerprint(value): + """Match the canonical JSON hash used by boundary approvals.""" + raw = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(raw.encode()).hexdigest() + + +def validation_policy_digest(spec, owner, registration, installation_id, slot_keys): + """Hash a live workspace policy with only registration names normalized.""" + if (not isinstance(spec, dict) or not isinstance(owner, str) or not owner + or not isinstance(registration, str) or not re.fullmatch(r"[a-f0-9]{16}", registration) + or not isinstance(installation_id, str) or not installation_id + or not isinstance(slot_keys, (list, tuple)) or not slot_keys + or any(not isinstance(slot, str) or not slot for slot in slot_keys) + or len(set(slot_keys)) != len(slot_keys)): + raise PreflightError("BoundaryApprovalInvalid") + text = json.dumps(spec, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + if "@SAW_" in text: + raise PreflightError("BoundaryApprovalInvalid") + text = text.replace("saw-ctl-" + registration, "@SAW_CONTROL_NS@") + text = text.replace(installation_id, "@SAW_INSTALLATION@") + for slot in slot_keys: + relay = (hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] + if len(slot_keys) > 1 else registration) + if len(slot_keys) > 1: + text = text.replace(owner + "-" + relay[:8], "@SAW_SLOT_" + slot + "@") + text = text.replace("gate-" + relay + "-", "gate-@SAW_RELAY_" + slot + "@-") + text = text.replace("ui-" + relay + "-browser-proof", + "ui-@SAW_RELAY_" + slot + "@-browser-proof") + return fingerprint(json.loads(text)) + + +def verify_boundary_hash(reference, key, observed): + """Fail with a fixed reason when a live policy differs from approval.""" + if not isinstance(reference, dict): + raise PreflightError("BoundaryApprovalMissing") + if key not in reference: + raise PreflightError("BoundaryApprovalMissing") + approved = reference.get(key) + if not isinstance(approved, str) or not re.fullmatch(r"[a-f0-9]{64}", approved): + raise PreflightError("BoundaryApprovalInvalid") + if approved != observed: + raise PreflightError("BoundaryApprovalChanged") + return True + + +def verify_boundary_reference(reference, mode): + """Require every policy approval before a new workspace can be reserved.""" + if mode not in {"container", "kata"}: + raise PreflightError("BoundaryApprovalInvalid") + if not isinstance(reference, dict): + raise PreflightError("BoundaryApprovalMissing") + keys = {"validation", "binding"} + if mode == "kata": + keys.update(("sandboxPlacement", "sandboxPlacementBinding")) + for key in keys: + approved = reference.get(key) + if approved is None: + raise PreflightError("BoundaryApprovalMissing") + if not isinstance(approved, str) or not re.fullmatch(r"[a-f0-9]{64}", approved): + raise PreflightError("BoundaryApprovalInvalid") + return True + + +def check_runtime_class(approved, observed): + """Check a RuntimeClass and return fixed, safe failure reasons.""" + if not isinstance(approved, dict) or not approved.get("name"): + raise PreflightError("RuntimeClassApprovalMissing") + if not isinstance(observed, dict): + raise PreflightError("RuntimeClassUnavailable") + metadata = observed.get("metadata") or {} + if not isinstance(metadata, dict): + raise PreflightError("RuntimeClassUnavailable") + if metadata.get("name") != approved["name"]: + raise PreflightError("RuntimeClassNameMismatch") + if not approved.get("uid") or metadata.get("uid") != approved["uid"]: + raise PreflightError("RuntimeClassIdentityMismatch") + if observed.get("handler") != "kata": + raise PreflightError("RuntimeClassHandlerMismatch") + labels = metadata.get("labels") or {} + if not isinstance(labels, dict): + raise PreflightError("RuntimeClassNotAttachable") + if labels.get("openshell.ai/sandbox-attachable") != "true": + raise PreflightError("RuntimeClassNotAttachable") + if not isinstance(observed.get("scheduling"), dict) or not observed["scheduling"]: + raise PreflightError("RuntimeClassSchedulingInvalid") + approved_digest = approved.get("sha256") + if not isinstance(approved_digest, str) or not re.fullmatch(r"[a-f0-9]{64}", approved_digest): + raise PreflightError("RuntimeClassApprovalInvalid") + if runtime_placement_digest(observed) != approved_digest: + raise PreflightError("RuntimeClassPlacementChanged") + return True + + +def check_observations(mode, observations): + """Require a complete fact set. Missing facts fail closed with safe codes. + + Values are booleans produced by a read-only cluster adapter. Do not pass + API responses, exception strings, URLs with credentials, or Secret data. + """ + if mode not in {"container", "kata"} or not isinstance(observations, dict): + raise PreflightError("InvalidPreflightRequest") + required = COMMON_CHECKS + (KATA_CHECKS if mode == "kata" else ()) + for check in required: + if observations.get(check) is not True: + raise PreflightError(CHECK_REASONS[check]) + return tuple(required) + + +def configuration_facts(site): + """Check site-owned inputs that do not require cluster access.""" + if not isinstance(site, dict): + return {key: False for key in ("network", "storage", "secrets", "registry", + "providerEgress", "revisions")} + stack = site.get("stack") if isinstance(site.get("stack"), dict) else {} + revisions = [item.get("revision") for item in stack.values() if isinstance(item, dict)] + required_components = ("identity", "policy", "proxy", "gateway", "audit") + revision_pin = all(isinstance(stack.get(name), dict) for name in required_components) and all( + isinstance(stack[name].get("revision"), str) + and re.fullmatch(r"[a-f0-9]{40}", stack[name]["revision"]) + for name in required_components) and bool(revisions) and len(revisions) == len(stack) and all( + isinstance(value, str) and re.fullmatch(r"[a-f0-9]{40}", value) for value in revisions) + enrollment = site.get("automaticEnrollment") or {} + if enrollment and not isinstance(enrollment, dict): + revision_pin = False + elif enrollment: + revision_pin = revision_pin and bool(re.fullmatch(r"[a-f0-9]{40}", str(enrollment.get("revision", "")))) + + image_values = [] + storage_values = [] + def walk(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + next_path = path + (str(key).lower(),) + if key == "image" and isinstance(child, str) and child: + image_values.append(child) + if key in {"storageClass", "storageClassName"} and isinstance(child, str) and child: + storage_values.append(child) + if key == "className" and "storage" in path and isinstance(child, str) and child: + storage_values.append(child) + walk(child, next_path) + elif isinstance(value, list): + for child in value: + walk(child, path) + walk(site) + + declared_hosts = site.get("providerAllowHosts") + policy = site.get("providerPolicy") + clean_hosts = (isinstance(declared_hosts, list) and bool(declared_hosts) + and all(isinstance(host, str) and host and "*" not in host + for host in declared_hosts) + and (policy is None or (isinstance(policy, dict) + and isinstance(policy.get("allowHosts"), list) + and all(isinstance(host, str) for host in policy["allowHosts"]) + and set(policy["allowHosts"]) == set(declared_hosts)))) + proxy = stack.get("proxy") or {} + proxy_values = proxy.get("values") or {} if isinstance(proxy, dict) else {} + bootstrap_egress = ((proxy_values.get("network") or {}).get("bootstrapEgress") + if isinstance(proxy_values, dict) else None) + storage_ok = bool(storage_values) and all(value.strip() for value in storage_values) + return { + "network": bool(stack.get("policy")) and isinstance(bootstrap_egress, list) + and bool(bootstrap_egress), + "storage": storage_ok, + "secrets": isinstance(site.get("secretStore", "vault-backend"), str) + and bool(site.get("secretStore", "vault-backend")), + "registry": bool(image_values) and all("@sha256:" in image for image in image_values), + "providerEgress": clean_hosts, + "revisions": revision_pin, + } + + +def storage_class_names(site): + names = set() + def walk(value, path=()): + if isinstance(value, dict): + for key, child in value.items(): + if key in {"storageClass", "storageClassName"} and isinstance(child, str) and child: + names.add(child) + elif key == "className" and "storage" in path and isinstance(child, str) and child: + names.add(child) + walk(child, path + (str(key).lower(),)) + elif isinstance(value, list): + for child in value: + walk(child, path) + walk(site) + return tuple(sorted(names)) + + +def verify_storage_classes(site, storage_classes): + names = storage_class_names(site) + available = {item.get("metadata", {}).get("name") for item in storage_classes + if isinstance(item, dict)} + default = any((item.get("metadata", {}).get("annotations") or {}).get( + "storageclass.kubernetes.io/is-default-class") == "true" + for item in storage_classes if isinstance(item, dict)) + if names and not set(names).issubset(available): + raise PreflightError("StorageUnavailable") + if not names and not default: + raise PreflightError("StorageUnavailable") + return names + + +def verify_secret_store(site, stores): + name = secret_store_name(site) + for store in stores: + metadata = store.get("metadata") or {} if isinstance(store, dict) else {} + if isinstance(metadata, dict) and metadata.get("name") == name and _ready(store): + return name + raise PreflightError("SecretServicesUnavailable") + + +def secret_store_name(site): + if not isinstance(site, dict): + return "vault-backend" + stack = site.get("stack") if isinstance(site.get("stack"), dict) else {} + identity = stack.get("identity") if isinstance(stack.get("identity"), dict) else {} + identity_values = identity.get("values") if isinstance(identity.get("values"), dict) else {} + return site.get("secretStore") or identity_values.get("secretStore") or "vault-backend" + + +def _ready(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and any( + isinstance(c, dict) and c.get("type") in {"Ready", "Completed"} and c.get("status") == "True" + for c in conditions) + + +def kata_config_ready(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + nodes = status.get("kataNodes") or {} + if not isinstance(nodes, dict): + return False + count = nodes.get("nodeCount") + ready_count = nodes.get("readyNodeCount") + if (type(count) is not int or count < 1 or type(ready_count) is not int + or ready_count != count): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and not any( + isinstance(condition, dict) and condition.get("type") == "InProgress" + and condition.get("status") == "True" for condition in conditions) + + +def kata_config_installed_nodes(obj): + if not isinstance(obj, dict): + return () + status = obj.get("status") or {} + if not isinstance(status, dict): + return () + nodes = status.get("kataNodes") or {} + if not isinstance(nodes, dict): + return () + installed = nodes.get("installed") + if not isinstance(installed, list): + return () + return tuple(name for name in installed if isinstance(name, str) and name) + + +def crd_established(obj): + if not isinstance(obj, dict): + return False + status = obj.get("status") or {} + if not isinstance(status, dict): + return False + conditions = status.get("conditions") or [] + return isinstance(conditions, list) and any( + isinstance(condition, dict) and condition.get("type") == "Established" + and condition.get("status") == "True" for condition in conditions) + + +def api_resources_available(discovery, group_version, resource_names): + """Check served API resources from discovery, including built-in APIs.""" + if not isinstance(discovery, dict) or discovery.get("groupVersion") != group_version: + return False + resources = discovery.get("resources") + if not isinstance(resources, list): + return False + available = {item.get("name") for item in resources if isinstance(item, dict)} + return set(resource_names).issubset(available) + + +def cluster_version_ready(obj): + try: + verify_cluster_version(obj) + return True + except PreflightError: + return False + + +def verify_cluster_version(obj): + if not isinstance(obj, dict): + raise PreflightError("ClusterVersionUnavailable") + status = obj.get("status") or {} + if not isinstance(status, dict): + raise PreflightError("ClusterVersionUnavailable") + conditions = status.get("conditions") or [] + available = isinstance(conditions, list) and any( + isinstance(condition, dict) and condition.get("type") == "Available" + and condition.get("status") == "True" for condition in conditions) + if not available: + raise PreflightError("ClusterVersionUnavailable") + desired = status.get("desired") or {} + version = desired.get("version") if isinstance(desired, dict) else None + if not isinstance(version, str): + raise PreflightError("ClusterVersionUnavailable") + version = version.strip() + match = re.fullmatch(r"(\d+)\.(\d+)(?:\.\d+)?(?:-[A-Za-z0-9.-]+)?", version) + if not match: + raise PreflightError("ClusterVersionUnavailable") + if tuple(map(int, match.groups())) < (4, 22): + raise PreflightError("UnsupportedOpenShiftVersion") + return version + + +def vault_health_ready(obj): + return isinstance(obj, dict) and obj.get("initialized") is True and obj.get("sealed") is False + + +def _matches_selector(labels, selector): + return bool(selector) and all(labels.get(key) == value for key, value in selector.items()) + + +def verify_kata_platform(site, runtime_class, kata_configs, nodes, api_readiness, + cluster_version): + """Check live Kata prerequisites from read-only Kubernetes API snapshots.""" + verify_cluster_version(cluster_version) + approved = site.get("kataRuntimeClass") if isinstance(site, dict) else None + check_runtime_class(approved, runtime_class) + if not isinstance(api_readiness, dict): + raise PreflightError("AgentSandboxUnavailable") + if api_readiness.get("kataconfigs.kataconfiguration.openshift.io") is not True: + raise PreflightError("SandboxedContainersUnavailable") + if not isinstance(kata_configs, list): + raise PreflightError("KataConfigNotReady") + ready_configs = [item for item in kata_configs + if isinstance(item, dict) and kata_config_ready(item)] + if not ready_configs: + raise PreflightError("KataConfigNotReady") + installed_nodes = {name for item in ready_configs for name in kata_config_installed_nodes(item)} + placement = (runtime_class.get("scheduling") or {}).get("nodeSelector") or {} + if not isinstance(placement, dict) or not placement: + raise PreflightError("RuntimeClassSchedulingInvalid") + eligible = [] + if not isinstance(nodes, list): + raise PreflightError("KataWorkersNotReady") + for node in nodes: + if not isinstance(node, dict): + continue + metadata = node.get("metadata") or {} + if not isinstance(metadata, dict): + continue + labels = metadata.get("labels") or {} + if not isinstance(labels, dict): + continue + roles = {key.removeprefix("node-role.kubernetes.io/") for key in labels + if key.startswith("node-role.kubernetes.io/")} + status = node.get("status") or {} + conditions = status.get("conditions", []) if isinstance(status, dict) else [] + ready = isinstance(conditions, list) and any( + isinstance(c, dict) and c.get("type") == "Ready" and c.get("status") == "True" + for c in conditions) + spec = node.get("spec") or {} + if (ready and isinstance(spec, dict) and not spec.get("unschedulable") + and labels.get("kubernetes.io/arch") == "amd64" + and "worker" in roles and not roles.intersection({"master", "control-plane"}) + and _matches_selector(labels, placement)): + eligible.append(metadata.get("name")) + if not eligible or not set(eligible).intersection(installed_nodes): + raise PreflightError("KataWorkersNotReady") + if api_readiness.get("sandboxes.agents.x-k8s.io") is not True: + raise PreflightError("AgentSandboxUnavailable") + if api_readiness.get("externalsecrets.external-secrets.io") is not True: + raise PreflightError("SecretServicesUnavailable") + if api_readiness.get("adminnetworkpolicies.policy.networking.k8s.io") is not True: + raise PreflightError("NetworkPolicyRejected") + if api_readiness.get("admissionregistration.k8s.io/v1") is not True: + raise PreflightError("AdmissionUnavailable") + return tuple(sorted(eligible)) diff --git a/charts/saw-users/files/profile-catalog.json b/charts/saw-users/files/profile-catalog.json index 2ca481760..dcb7aef9d 100644 --- a/charts/saw-users/files/profile-catalog.json +++ b/charts/saw-users/files/profile-catalog.json @@ -28,6 +28,7 @@ { "enabled": true, "harnessRequired": false, + "image": "quay.io/aipcc/base-images/agentic/openclaw@sha256:896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", "name": "notebook", "type": "openclaw", "uiRoute": true @@ -35,6 +36,7 @@ { "enabled": false, "harnessRequired": false, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", "name": "cuda-sandbox", "type": "nemoclaw", "uiRoute": false @@ -42,6 +44,7 @@ { "enabled": false, "harnessRequired": false, + "image": "base", "name": "toolbox", "type": "generic", "uiRoute": false @@ -51,6 +54,9 @@ ] }, "custom-inference": { + "dedicatedEndpointSecrets": [ + "inference" + ], "description": "Agent workspace backed by a custom OpenAI-compatible endpoint (vLLM, Ollama, ...)", "harnessRequired": false, "secrets": { @@ -103,7 +109,10 @@ "sandboxes": [ { "enabled": true, + "endpointInput": "inference.url", "harnessRequired": false, + "image": "quay.io/aipcc/base-images/agentic/openclaw@sha256:896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", + "modelInput": "inference.model", "name": "notebook", "type": "openclaw", "uiRoute": true @@ -168,6 +177,8 @@ { "enabled": true, "harnessRequired": false, + "image": "quay.io/aipcc/base-images/agentic/openclaw@sha256:896d1526ee29b53167034ba65d1c758c8da777f441ed094e19f3bcf6d7445c61", + "modelInput": "inference.model", "name": "notebook", "type": "openclaw", "uiRoute": true @@ -175,6 +186,7 @@ { "enabled": false, "harnessRequired": false, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", "name": "cuda-sandbox", "type": "nemoclaw", "uiRoute": false @@ -182,6 +194,7 @@ { "enabled": false, "harnessRequired": false, + "image": "base", "name": "toolbox", "type": "generic", "uiRoute": false @@ -196,6 +209,8 @@ { "enabled": true, "harnessRequired": false, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", + "modelInput": "inference.cuda_model", "name": "cuda-sandbox", "type": "nemoclaw", "uiRoute": true @@ -203,6 +218,7 @@ { "enabled": false, "harnessRequired": false, + "image": "base", "name": "toolbox", "type": "generic", "uiRoute": false @@ -286,6 +302,7 @@ { "enabled": true, "harnessRequired": true, + "image": "ghcr.io/validatedpatterns-sandbox/nemoclaw-sandbox@sha256:28457eec59e71eca310cd72fc4b57c71506c3fce4c30f2636a237dca85725dbd", "name": "assistant", "type": "nemoclaw", "uiRoute": true diff --git a/charts/saw-users/files/projector.py b/charts/saw-users/files/projector.py new file mode 100644 index 000000000..23282e0eb --- /dev/null +++ b/charts/saw-users/files/projector.py @@ -0,0 +1,250 @@ +"""Read-only ApplicationSet projection for installations without Developer Hub. + +The same workspace.project_workspace function is used by the RHDH adapter. +Only named public enrollment publications are read from Kubernetes. Desired +workspace inputs come from the mounted, operator-owned configuration. +""" +import hmac +import http.server +import json +import os +from pathlib import Path +import re +import ssl +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, HTTPSHandler, Request, build_opener + +import workspace +import preflight + +LIMIT = 4 * 1024 * 1024 + + +class ProjectionError(ValueError): + pass + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +class Publications: + def __init__(self, service_account): + self.root = Path(service_account) + self.opener = build_opener(NoRedirect(), HTTPSHandler( + context=ssl.create_default_context(cafile=str(self.root / "ca.crt")))) + + def __call__(self, path): + if not re.fullmatch(r"/api/v1/namespaces/saw-ctl-[a-f0-9]{16}/configmaps/[a-z0-9-]+", path): + raise ProjectionError("publication is outside the workspace scope") + return self.get(path, publication=True) + + def preflight_resource(self, path): + fixed = { + "/apis/config.openshift.io/v1/clusterversions/version", + "/apis/kataconfiguration.openshift.io/v1/kataconfigs", + "/api/v1/nodes", + "/apis/admissionregistration.k8s.io/v1", + } + allowed = (path in fixed or re.fullmatch( + r"/api/v1/namespaces/saw-[a-z0-9-]+", path) or re.fullmatch( + r"/apis/node.k8s.io/v1/runtimeclasses/[a-z0-9-]+", path) or re.fullmatch( + r"/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + r"(?:kataconfigs.kataconfiguration.openshift.io|sandboxes.agents.x-k8s.io|" + r"externalsecrets.external-secrets.io|adminnetworkpolicies.policy.networking.k8s.io)", path) or re.fullmatch( + r"/apis/admissionregistration.k8s.io/v1/" + r"(?:validatingadmissionpolicies|validatingadmissionpolicybindings)/" + r"saw-saw-[a-z0-9-]+-runtime(?:-kata)?", path)) + if not allowed: + raise ProjectionError("preflight resource is outside the workspace scope") + return self.get(path) + + def get(self, path, *, publication=False): + token = (self.root / "token").read_text().strip() + request = Request("https://kubernetes.default.svc" + path, + headers={"Authorization": "Bearer " + token}) + try: + with self.opener.open(request, timeout=20) as response: + body = response.read(LIMIT + 1) + if len(body) > LIMIT: + raise ProjectionError("workspace API response exceeds the size limit") + return json.loads(body) + except HTTPError as exc: + exc.close() + if exc.code == 404 or publication and exc.code == 403: + return {} + raise ProjectionError("workspace API request failed") from None + except (URLError, OSError, ValueError): + raise ProjectionError("workspace API is unavailable or invalid") from None + + def previous(self, owner): + ns = os.environ["ARGO_NAMESPACE"] + appset = os.environ["PROJECTOR_NAME"] + app = self.get(f"/apis/argoproj.io/v1alpha1/namespaces/{ns}/applications/{appset}-{owner['name']}") + return workspace.previous_child(app, appset, owner) + + +def verify_gitops_preflight(owner, site, mode, read): + """Stop ApplicationSet generation on a changed live Kata boundary.""" + automatic = site.get("automaticEnrollment", {}) + if not automatic or automatic.get("enabled") is False: + return + reference = automatic.get("reference") + preflight.verify_boundary_reference(reference, mode) + if mode == "kata": + approval = site.get("kataRuntimeClass") or {} + if not isinstance(approval, dict) or not approval.get("name"): + raise preflight.PreflightError("RuntimeClassApprovalMissing") + try: + version = read("/apis/config.openshift.io/v1/clusterversions/version") + runtime = read("/apis/node.k8s.io/v1/runtimeclasses/" + approval["name"]) + configs = read("/apis/kataconfiguration.openshift.io/v1/kataconfigs") + nodes = read("/api/v1/nodes") + readiness = {} + for name in ("kataconfigs.kataconfiguration.openshift.io", + "sandboxes.agents.x-k8s.io", + "externalsecrets.external-secrets.io", + "adminnetworkpolicies.policy.networking.k8s.io"): + readiness[name] = preflight.crd_established(read( + "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + name)) + readiness["admissionregistration.k8s.io/v1"] = preflight.api_resources_available( + read("/apis/admissionregistration.k8s.io/v1"), + "admissionregistration.k8s.io/v1", + ("validatingadmissionpolicies", "validatingadmissionpolicybindings")) + preflight.verify_kata_platform(site, runtime, (configs or {}).get("items"), + (nodes or {}).get("items"), readiness, version) + except preflight.PreflightError: + raise + except Exception: + raise preflight.PreflightError("KataPlatformUnavailable") from None + + name = owner["name"] + namespace_name = "saw-" + name + try: + namespace = read("/api/v1/namespaces/" + namespace_name) + except Exception: + raise preflight.PreflightError("BoundaryNamespaceUnavailable") from None + if not namespace: + return # Enrollment verifies policies after the first namespace exists. + metadata = namespace.get("metadata", {}) + labels = metadata.get("labels", {}) + if (metadata.get("deletionTimestamp") or labels.get("saw.redhat.com/runtime-owner") != owner["installationID"] + or labels.get("saw.redhat.com/registration") != owner["registration"]): + raise preflight.PreflightError("BoundaryApprovalIdentityMismatch") + policy = "saw-" + namespace_name + "-runtime" + resources = [("validation", "validatingadmissionpolicies", policy), + ("binding", "validatingadmissionpolicybindings", policy)] + if mode == "kata": + resources.extend((("sandboxPlacement", "validatingadmissionpolicies", policy + "-kata"), + ("sandboxPlacementBinding", "validatingadmissionpolicybindings", policy + "-kata"))) + for key, plural, resource_name in resources: + try: + obj = read("/apis/admissionregistration.k8s.io/v1/" + plural + "/" + resource_name) + except Exception: + raise preflight.PreflightError("BoundaryPolicyUnavailable") from None + if not isinstance(obj, dict) or not isinstance(obj.get("spec"), dict): + raise preflight.PreflightError("BoundaryPolicyUnavailable") + digest = (preflight.validation_policy_digest( + obj["spec"], name, owner["registration"], owner["installationID"], + sorted(site["targets"])) if key == "validation" else preflight.fingerprint(obj["spec"])) + try: + preflight.verify_boundary_hash(reference, key, digest) + except preflight.PreflightError as exc: + raise preflight.PreflightError(exc.reason + ":" + key) from None + if plural == "validatingadmissionpolicies": + status = obj.get("status", {}) + type_checking = status.get("typeChecking", {}) if isinstance(status, dict) else {} + if (not isinstance(type_checking, dict) + or status.get("observedGeneration") != obj.get("metadata", {}).get("generation") + or "typeChecking" not in status or type_checking.get("expressionWarnings")): + raise preflight.PreflightError("AdmissionUnavailable:" + key) + + +def parameters(config, catalog, read): + """Project the whole desired set atomically; errors never mean deletion.""" + if not isinstance(config, dict) or not isinstance(config.get("users"), list): + raise ProjectionError("workspace configuration requires a users list") + defaults = {k: v for k, v in config.items() if k not in ("users", "containerSites", "projection")} + # Child charts render a fully projected workspace, not another generator. + defaults["projection"] = {"enabled": False} + # Container entries have a pre-resolved subject. VM owner resolution stays + # with the root chart and must not create a second owner of its resources. + defaults["ownerResolution"] = {**defaults.get("ownerResolution", {}), "enabled": False} + result, seen = [], set() + sites = config.get("containerSites", {}) + for owner in config["users"]: + if not isinstance(owner, dict): + raise ProjectionError("workspace entries must be mappings") + name = owner.get("name", "") + if not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,17}[a-z0-9])?", name) or name in seen: + raise ProjectionError("workspace names must be unique DNS labels of at most 19 characters") + seen.add(name) + if owner.get("values", {}).get("deployment", {}).get("mode", "vm") not in {"container", "kata"}: + raise ProjectionError("standalone container projection received another backend") + site = sites.get(name) + try: + profile = (owner.get("profiles") or [None])[0] + preflight.verify_profile_image_access(site, profile, catalog) + values = workspace.project_workspace(owner, defaults, site, catalog, read, getattr(read, "previous", None)) + live_read = getattr(read, "preflight_resource", None) + if callable(live_read): + mode = owner["values"]["deployment"]["mode"] + verify_gitops_preflight(owner, values["containerSites"][name], mode, live_read) + except preflight.PreflightError as exc: + raise ProjectionError("workspace " + name + ": preflight failed: " + exc.reason) from None + except workspace.WorkspaceError as exc: + raise ProjectionError(str(exc)) from None + except (KeyError, TypeError, ValueError): + raise ProjectionError("workspace configuration or enrollment is invalid") from None + # `values` is reserved by the ApplicationSet plugin protocol. + result.append({"name": name, "revision": values.get("global", {}).get("targetRevision", ""), + "workspaceValues": json.dumps(values, sort_keys=True)}) + return result + + +def server(config_file, catalog_file, token_file, read, address): + class Handler(http.server.BaseHTTPRequestHandler): + def send(self, code, value): + body = json.dumps(value).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self.send(200 if self.path == "/healthz" else 404, {"status": "running"}) + + def do_POST(self): + if self.path != "/api/v1/getparams.execute": + return self.send(404, {"error": "not found"}) + try: + token = Path(token_file).read_text().strip() + except OSError: + return self.send(503, {"error": "plugin identity is unavailable"}) + if not token or not hmac.compare_digest(self.headers.get("Authorization", ""), "Bearer " + token): + return self.send(403, {"error": "forbidden"}) + try: + config = json.loads(Path(config_file).read_text()) + catalog = json.loads(Path(catalog_file).read_text())["profiles"] + output = parameters(config, catalog, read) + except ProjectionError as exc: + return self.send(503, {"error": str(exc)}) + except (OSError, ValueError, KeyError, TypeError): + # Returning an empty list on error would prune live workspaces. + return self.send(503, {"error": "workspace projection is not ready"}) + self.send(200, {"output": {"parameters": output}}) + + def log_message(self, fmt, *args): + # Avoid logging request headers, inputs or publication values. + print("workspace projection request completed", flush=True) + + return http.server.ThreadingHTTPServer(address, Handler) + + +if __name__ == "__main__": + with server("/config/workspaces.json", "/source/profile-catalog.json", "/identity/token", + Publications("/var/run/secrets/kubernetes.io/serviceaccount"), + ("0.0.0.0", int(os.environ.get("PORT", "4355")))) as listener: + listener.serve_forever() diff --git a/charts/saw-users/files/projector_identity.py b/charts/saw-users/files/projector_identity.py new file mode 100644 index 000000000..b191a6cdb --- /dev/null +++ b/charts/saw-users/files/projector_identity.py @@ -0,0 +1,124 @@ +"""Create-once bearer identity for the internal ApplicationSet plugin. + +Only the two configured Secret names are read. Both projections are immutable; +namespace identities and ownership are checked before an existing value is used. +""" +import base64 +import hashlib +import json +import secrets +from pathlib import Path +import re +import ssl +from urllib.error import HTTPError +from urllib.request import HTTPRedirectHandler, HTTPSHandler, Request, build_opener + + +class IdentityError(ValueError): + pass + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +class Api: + def __init__(self): + self.root = Path("/var/run/secrets/kubernetes.io/serviceaccount") + self.opener = build_opener(NoRedirect(), HTTPSHandler( + context=ssl.create_default_context(cafile=str(self.root / "ca.crt")))) + + def call(self, method, path, body=None): + raw = json.dumps(body).encode() if body is not None else None + token = (self.root / "token").read_text().strip() + req = Request("https://kubernetes.default.svc"+path, data=raw, method=method, + headers={"Content-Type": "application/json", "Authorization": "Bearer "+token}) + try: + with self.opener.open(req, timeout=20) as response: + value = response.read(1024*1024+1) + if len(value) > 1024*1024: + raise IdentityError("oversized identity response") + return json.loads(value) + except HTTPError as exc: + status = exc.code + exc.close() + if (method == "GET" and status == 404) or (method == "POST" and status == 409): + return None + raise IdentityError("identity API request failed") from None + + +def provision(config, api): + if set(config) != {"namespace", "argoNamespace", "name", "owner"}: + raise IdentityError("invalid identity configuration") + for value in config.values(): + if not isinstance(value, str) or not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", value): + raise IdentityError("invalid identity name") + if config["namespace"] == config["argoNamespace"]: + raise IdentityError("projector must have a dedicated namespace") + namespaces = [config["namespace"], config["argoNamespace"]] + identities = {} + for ns in namespaces: + obj = api.call("GET", "/api/v1/namespaces/"+ns) or {} + meta = obj.get("metadata", {}) + if not meta.get("uid") or meta.get("deletionTimestamp"): + raise IdentityError("identity namespace is not ready") + identities[ns] = meta["uid"] + owners = {} + for ns, group, kind, plural in [(config["namespace"], "apps/v1", "Deployment", "deployments"), + (config["argoNamespace"], "argoproj.io/v1alpha1", "ApplicationSet", "applicationsets")]: + obj = api.call("GET", f"/apis/{group}/namespaces/{ns}/{plural}/{config['name']}") or {} + meta = obj.get("metadata", {}) + if (not meta.get("uid") or meta.get("deletionTimestamp") + or meta.get("labels", {}).get("saw.redhat.com/projector-owner") != config["owner"]): + raise IdentityError("plugin identity owner is not ready") + owners[ns] = [{"apiVersion": group, "kind": kind, "name": config["name"], "uid": meta["uid"]}] + revision = hashlib.sha256(json.dumps({"config": config, "namespaces": identities, "owners": owners}, sort_keys=True).encode()).hexdigest() + paths = [f"/api/v1/namespaces/{ns}/secrets/{config['name']}" for ns in namespaces] + + def checked(obj, ns): + meta = (obj or {}).get("metadata", {}) + data = (obj or {}).get("data", {}) + if (meta.get("deletionTimestamp") or meta.get("ownerReferences") != owners[ns] + or meta.get("labels", {}).get("saw.redhat.com/projector-owner") != config["owner"] + or meta.get("annotations", {}).get("saw.redhat.com/identity-sha256") != revision + or obj.get("immutable") is not True or obj.get("type") != "Opaque" + or set(data) != {"token"}): + raise IdentityError("existing plugin identity is foreign or changed") + try: + decoded = base64.b64decode(data["token"], validate=True).decode("ascii") + except (ValueError, UnicodeError): + raise IdentityError("invalid plugin identity") from None + if not re.fullmatch(r"[A-Za-z0-9_-]{43}", decoded): + raise IdentityError("invalid plugin token") + return data + + first = api.call("GET", paths[0]) + if first is None: + if any(api.call("GET", path) for path in paths[1:]): + raise IdentityError("orphaned plugin identity; refusing automatic replacement") + token = base64.b64encode(secrets.token_urlsafe(32).encode()).decode() + data = {"token": token} + else: + data = checked(first, namespaces[0]) + for ns, path in zip(namespaces, paths): + expected = {"apiVersion": "v1", "kind": "Secret", "type": "Opaque", "immutable": True, + "metadata": {"name": config["name"], "namespace": ns, "ownerReferences": owners[ns], + "labels": {"saw.redhat.com/projector-owner": config["owner"], + "app.kubernetes.io/part-of": "argocd"}, + "annotations": {"saw.redhat.com/identity-sha256": revision}}, + "data": data} + if api.call("GET", path) is None: + api.call("POST", f"/api/v1/namespaces/{ns}/secrets", expected) + observed = checked(api.call("GET", path), ns) + if ns == namespaces[0]: + data = observed # use the winner of a concurrent primary creation + elif observed != data: + raise IdentityError("plugin identity projections differ") + + +if __name__ == "__main__": + try: + provision(json.loads(Path("/source/identity.json").read_text()), Api()) + except Exception: + raise SystemExit("plugin identity initialization failed; no existing identity was replaced") from None diff --git a/charts/saw-users/files/workspace.py b/charts/saw-users/files/workspace.py new file mode 100644 index 000000000..bc86b4d01 --- /dev/null +++ b/charts/saw-users/files/workspace.py @@ -0,0 +1,410 @@ +"""Common workspace projection used by installation and portal adapters. + +Inputs are operator configuration, a BOM catalog and an immutable public +installation binding. Callers own authentication, credential storage, reads and +resource application. This module never discovers or changes cluster state. +""" +import hashlib +import json +import re +import urllib.parse +import urllib.error + + +class WorkspaceError(ValueError): + """Invalid workspace configuration or immutable publication.""" + + +SAFE_FAILURE_REASONS = frozenset({ + "ProfileRejected", "InvalidPlan", "PlacementRejected", "IdentityConflict", + "InvalidIntent", "StaleIntent", "ReconcilerStateRejected", "GatewayMismatch", + "ProviderProfileMismatch", "WorkspaceRejected", "ProviderRejected", + "SandboxRejected", "SandboxStateMismatch", "SandboxDrift", "CoreRejected", + "CoreFailure", +}) +SAFE_RPC_CODES = frozenset({ + "NOT_FOUND", "ALREADY_EXISTS", "UNAVAILABLE", "DEADLINE_EXCEEDED", "ABORTED", + "RESOURCE_EXHAUSTED", "PERMISSION_DENIED", "UNAUTHENTICATED", "INVALID_ARGUMENT", + "FAILED_PRECONDITION", "UNKNOWN", +}) + + +def enabled_catalog_sandboxes(profile): + for workspace in profile.get("workspaces", []): + if workspace.get("enabled"): + for sandbox in workspace.get("sandboxes", []): + if sandbox.get("enabled"): + yield workspace["name"], sandbox + + +def sandbox_targets(user, registration, profile, *, model_lifecycle=False): + """Expand the shared BOM catalog into isolated Kubernetes attachments.""" + enabled = list(enabled_catalog_sandboxes(profile)) + targets = {} + for workspace, sb in enabled: + key = workspace + "/" + sb["name"] + suffix = hashlib.sha256((registration+"/"+key).encode()).hexdigest()[:16] + single = len(enabled)==1 + targets[key] = dict(workspace=workspace, sandbox=sb["name"], ui=bool(sb.get("uiRoute")), + image=sb.get("image", ""), base=user if single else user+"-"+suffix[:8], + relayID=registration if single else suffix, + subjectMap="saw-portal-relay-subject" if single else "saw-portal-relay-"+suffix) + if model_lifecycle and sb.get("modelInput"): + targets[key]["modelInput"] = sb["modelInput"] + return targets + + +def public_model_inputs(profile, secrets): + """Only catalog-declared, noncredential model selectors can enter public intent.""" + values = {} + confidential = {secrets.get(name, {}).get(field["key"]) + for name, entry in profile.get("secrets", {}).items() + for field in entry["fields"] if field["kind"] == "secret"} + custom_models = {sb.get("modelInput") for _, sb in enabled_catalog_sandboxes(profile) if sb.get("endpointInput")} + for key in {sb["modelInput"] for _, sb in enabled_catalog_sandboxes(profile) if sb.get("modelInput")}: + secret, field = key.split(".",1) + definition = next((f for f in profile["secrets"][secret]["fields"] + if f["key"] == field and f["kind"] == "text"), None) + if definition is None: + raise WorkspaceError("invalid public model selector") + value = secrets.get(secret, {}).get(field) + if value is None or value == "": + continue + if not isinstance(value, str) or value in confidential: + raise WorkspaceError("invalid public model selector") + # An official provider may publish a qualified model ID as this field's + # BOM default. Other qualified IDs require the custom endpoint contract. + qualified = key in custom_models or value == definition.get("default") + spelling = r"[A-Za-z0-9][A-Za-z0-9._/-]{0,99}" if qualified else r"[A-Za-z0-9][A-Za-z0-9._-]{0,99}" + if not re.fullmatch(spelling, value): + raise WorkspaceError("invalid public model selector") + values[key] = value + return values + + +def approved_endpoint(profile, secrets, boundary): + refs = {sb["endpointInput"] for _, sb in enabled_catalog_sandboxes(profile) if sb.get("endpointInput")} + if not refs: + return None + endpoint = boundary.get("inferenceBaseURL", "") + if not isinstance(endpoint, str) or not re.fullmatch( + r"https://[a-z0-9-]+(?:\.[a-z0-9-]+)+(?::443)?(?:/[A-Za-z0-9_-]+)*/v1", endpoint): + raise WorkspaceError("site has no approved custom inference endpoint") + if boundary.get("allowHosts") != [urllib.parse.urlsplit(endpoint).hostname]: + raise WorkspaceError("custom inference requires a dedicated broker host") + for ref in refs: + secret, key = ref.split(".", 1) + fields = profile["secrets"][secret]["fields"] + if not any(f["key"] == key and f["kind"] == "url" for f in fields): + raise WorkspaceError("endpoint input must be public URL metadata") + if secrets.get(secret, {}).get(key, "").rstrip("/") != endpoint: + raise WorkspaceError("endpoint differs from the site-approved custom inference URL") + return endpoint + + +def enrolled_site(site, ws, read): + """Only the trusted bootstrap publishes bindings; images, code and endpoints always come from Git.""" + a = site.get("automaticEnrollment") + if not a or not read: + return site + reg, install = ws["registration"], ws["installationID"] + ns = "saw-ctl-" + reg + def publication(name): + obj = read(f"/api/v1/namespaces/{ns}/configmaps/{name}") + if not obj or not obj.get("data"): + return {} + meta=obj["metadata"] + if obj.get("immutable") is not True or meta.get("deletionTimestamp") or meta.get("labels",{}).get("saw.redhat.com/runtime-owner")!=install: + raise WorkspaceError("invalid registration publication") + return obj["data"] + fresh = {k:v for k,v in site.items() if k!="enrollment"} + data=publication("saw-portal-enrollment") + if a.get("enabled") is False: + if data: + raise WorkspaceError("cannot disable enrollment of a bound workspace; use desiredState") + return fresh + for key, kind in (("reference", dict), ("relay", dict), ("revision", str), + ("image", str), ("buildNamespace", str)): + if not isinstance(a.get(key), kind) or not a[key]: + raise WorkspaceError(f"container site for {ws['name']} requires automaticEnrollment.{key}") + if not data: + return fresh + bound=json.loads(data["binding.json"]) + if set(bound)!={"registration","intent","approved","gatewayCA"} or bound["registration"]!=reg or set(bound["approved"])!=set(a["reference"]): + raise WorkspaceError("invalid published enrollment") + intent=bound["intent"] + if intent["bindings"]["ownerID"]!=install or intent["bindings"]["namespace"]!="saw-"+ws["name"] or intent["bindings"]["ownerSubject"]!=ws["ownerSubject"] or intent["profileRef"]["sha256"]!=site.get("selectedDigest",a.get("profileSHA256")): + raise WorkspaceError("published intent does not belong to this workspace") + subjects = {target["subjectMap"]:publication(target["subjectMap"]).get("subject", "") + for target in site.get("targets",{}).values() if target["ui"]} + fresh["subjects"] = subjects + sub=subjects.get("saw-portal-relay-subject","") if site.get("targets") else publication("saw-portal-relay-subject").get("subject","") + if sub and (not re.fullmatch(r"[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}",sub) or sub==ws["ownerSubject"]): + raise WorkspaceError("invalid relay subject") + relay={**a["relay"],"gatewayCA":bound["gatewayCA"],"subject":sub,"workspaceID":""} + fresh["enrollment"]={**{k:a[k] for k in ("revision","image","buildNamespace")},"registration":reg,"intent":intent,"approved":bound["approved"],"relay":relay} + return fresh + + +def previous_child(application, appset, owner): + """Read only the public values of the child controlled by this ApplicationSet.""" + if not application: + return None + meta = application["metadata"] + if (meta.get("deletionTimestamp") or meta.get("labels", {}).get("openshell.pattern/owner") != owner["name"] + or not any(ref.get("apiVersion") == "argoproj.io/v1alpha1" and ref.get("kind") == "ApplicationSet" + and ref.get("name") == appset and ref.get("controller") is True for ref in meta.get("ownerReferences", []))): + raise WorkspaceError("cannot verify existing workspace Application ownership") + source = application["spec"]["source"] + if source.get("path") != "charts/saw-users": + raise WorkspaceError("unexpected workspace Application source") + values = json.loads(source["helm"]["values"]) + users = values["users"] + if len(users) != 1 or any(users[0].get(k) != owner.get(k) for k in ("name", "installationID", "registration", "ownerSubject")): + raise WorkspaceError("existing Application belongs to a different registration") + if not isinstance(values, dict) or values["global"]["targetRevision"] != source["targetRevision"]: + raise WorkspaceError("existing Application revision does not match its values") + return values + + +def validate_runtime_placement(mode, site): + runtime = site.get("kataRuntimeClass") + if mode == "kata": + if (not isinstance(runtime, dict) or set(runtime) != {"name", "uid", "sha256"} + or not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", str(runtime["name"])) + or not re.fullmatch(r"[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}", str(runtime["uid"])) + or not re.fullmatch(r"[a-f0-9]{64}", str(runtime["sha256"]))): + raise WorkspaceError("kata requires a reviewed site RuntimeClass") + elif runtime is not None: + raise WorkspaceError("container registration cannot use a Kata approval") + + +def project_workspace(workspace, defaults, site, catalog, read=None, previous=None): + try: + values = _project_workspace(workspace, defaults, site, catalog, read) + name = workspace["name"] + projected = values.get("containerSites", {}).get(name, {}) + if previous and projected and site.get("automaticEnrollment"): + subjects = projected.get("subjects", {}) + incomplete = not projected.get("enrollment") or any( + target.get("ui") and not subjects.get(target["subjectMap"]) for target in projected["targets"].values()) + if incomplete: + retained = previous(workspace) + if retained: + old = retained["containerSites"][name] + if (old.get("enrollment") and not projected.get("enrollment") or + any(sub and not subjects.get(key) for key, sub in old.get("subjects", {}).items())): + # Returning the exact existing parameters holds only this child; + # omitting it could tell the static ApplicationSet to prune it. + return retained + return values + except WorkspaceError: + raise + except (KeyError, TypeError, ValueError, AttributeError) as exc: + # Inputs include operator configuration and immutable API publications. + # Do not include arbitrary values (especially JSON bodies) in diagnostics. + detail = f"missing {exc.args[0]}" if isinstance(exc, KeyError) and exc.args[0] in { + "subject", "registration", "installationID", "reference", "allowHosts", "bindings", + "binding.json", "profileRef", "sha256", "initialModelInputs"} else "invalid site or publication" + raise WorkspaceError(f"workspace {workspace.get('name', '?')}: {detail}") from None + + +def _project_workspace(workspace, defaults, site, catalog, read=None): + """Project an authenticated registration into common saw-users chart values. + + Entry points resolve the owner, choose a registration and load any public + enrollment objects. They must not manufacture a successful enrollment. + Neither RHDH nor a particular Kubernetes client is required here. + """ + ws = {**workspace, "values": {**workspace.get("values", {})}} + mode = ws["values"].get("deployment", {}).get("mode", "vm") + if mode not in {"vm", "container", "kata"}: + raise WorkspaceError("unsupported workspace deployment mode") + ws["values"]["deployment"] = {"mode": mode} + # This map belongs to the operator's installation configuration, never the + # registration or an owner request. Pin only code in the configured repository. + revisions = defaults.get("workspaceRevisions", {}) + if not isinstance(revisions, dict) or any( + not isinstance(name, str) or not re.fullmatch(r"[a-z0-9](?:[-a-z0-9]{0,17}[a-z0-9])?", name) + or not isinstance(revision, str) or not re.fullmatch(r"[a-f0-9]{40}", revision) + for name, revision in revisions.items()): + raise WorkspaceError("workspaceRevisions requires owner names and full Git commit SHAs") + values = {**{k:v for k,v in defaults.items() if k != "workspaceRevisions"}, "users": [ws]} + if ws.get("name") in revisions: + values["global"] = {**defaults.get("global", {}), "targetRevision": revisions[ws["name"]]} + if mode == "vm": + return values + if not isinstance(site, dict) or not site: + raise WorkspaceError(f"container site is missing for {ws['name']}") + if ws.get("ownerSubject") != site.get("subject"): + raise WorkspaceError("container site must bind the resolved owner subject") + validate_runtime_placement(mode, site) + automatic = site.get("automaticEnrollment", {}) + selected, *extra = ws.get("profiles", []) or [None] + foundation = automatic.get("enabled") is False + if extra or selected not in catalog or not foundation and selected not in automatic.get("approvedProfiles", {}): + raise WorkspaceError("container requires one approved BOM profile") + if not ws.get("registration") or not ws.get("installationID"): + raise WorkspaceError("container requires a registration and installation identity") + overrides = site.get("profileImageOverrides", {}).get(selected, {}) + if overrides: + ws["imageOverrides"] = overrides + managed = ws.get("lifecycleVersion") == 1 + site = {**site, + "targets": sandbox_targets(ws["name"], ws["registration"], catalog[selected], + model_lifecycle=managed), + "selectedDigest": automatic.get("approvedProfiles", {}).get(selected, ""), + "modelInputs": ws.get("modelInputs", {})} + for target in site["targets"].values(): + target["image"] = overrides.get(target["sandbox"], target["image"]) + boundary = site.get("profileBoundaries", {}).get(selected) + if boundary: + site = {**site, "providerAllowHosts": boundary["allowHosts"], + "automaticEnrollment": {**automatic, "reference": boundary["reference"]}} + if ws.get("customInferenceURL"): + if not boundary or boundary.get("inferenceBaseURL") != ws["customInferenceURL"]: + raise WorkspaceError("endpoint differs from the site-approved custom inference URL") + site["customInferenceURL"] = ws["customInferenceURL"] + if managed: + site.update(lifecycleVersion=1, modelInputs=ws["initialModelInputs"], + requestedModels=ws.get("modelInputs", {}), desiredState=ws["desiredState"], + restartGeneration=ws["restartGeneration"]) + values["containerSites"] = {ws["name"]: enrolled_site(site, ws, read)} + return values + + +# Preserve the existing lifecycle API stage IDs across entry points. +STAGE_IDS = ("registered", "apps", "vm", "running", "ready") + + +def sandbox_ui_links(user, profiles, catalog, domain): + out = [] + for profile in profiles: + for workspace, sb in enabled_catalog_sandboxes(catalog.get(profile, {})): + if sb.get("uiRoute"): + out.append((f"{sb['name']} UI ({workspace})", + f"https://{user}-{workspace}-{sb['name']}-ui.apps.{domain}")) + return out + + + +def container_status(read_resource, entry, site, argo_ns, domain, ping, catalog=None): + # Reuse the VM pipeline's wire stage IDs so old RHDH tasks can finish. + # Only observations and their user-facing names differ. + user = entry["name"] + ns = f"saw-{user}" + reg = entry["registration"] + rev = entry.get("intentGeneration") + gen = isinstance(rev, int) and not isinstance(rev, bool) and rev > 0 + app = read_resource(f"/apis/argoproj.io/v1alpha1/namespaces/{argo_ns}/applications/saw-{user}") or {} + meta = app.get("metadata", {}) + app_ok = meta.get("labels", {}).get("openshell.pattern/owner") == user and not meta.get("deletionTimestamp") + app_status = app.get("status", {}) + def read(name, field): + cm = read_resource(f"/api/v1/namespaces/{ns}/configmaps/{name}") or {} + meta = cm.get("metadata", {}) + labels = meta.get("labels", {}) + if (meta.get("deletionTimestamp") or not reg + or labels.get("saw.redhat.com/runtime-owner") != entry["installationID"] + or field == "status.json" and labels.get("saw.redhat.com/managed-by") != "kubernetes-runtime-controller"): + return {} + try: + value = json.loads(cm.get("data", {}).get(field, "{}")) + return value if isinstance(value, dict) else {} + except (ValueError, TypeError): + return {} + intent = read("saw-runtime-intent", "intent.json") if app_ok else {} + bound = bool(reg and site and gen and intent.get("generation") == rev + and intent.get("bindings", {}).get("ownerSubject") == site["subject"] + and intent.get("bindings", {}).get("namespace") == ns + and intent.get("bindings", {}).get("mode") == entry.get("values", {}).get("deployment", {}).get("mode", "container") + and intent.get("bindings", {}).get("ownerID") == entry["installationID"]) + desired = entry.get("desiredState", "Running") + bound = bound and intent.get("desiredState") == desired + st = read("saw-runtime-status", "status.json") if bound else {} + profile = (read_resource(f"/api/v1/namespaces/{ns}/configmaps/saw-bom-profiles") or {}) if bound else {} + files = profile.get("data", {}) + def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + ref = intent.get("profileRef", {}) + current = (bound and not profile.get("metadata", {}).get("deletionTimestamp") + and ref.get("name") == "saw-bom-profiles" and bool(ref.get("uid")) + and ref["uid"] == profile.get("metadata", {}).get("uid") and ref.get("sha256") == digest(files) + and type(st.get("observedGeneration")) is int and st["observedGeneration"] == rev + and st.get("inputRevision") == digest({"intent": intent, "files": files})) + running = current and st.get("phase") == "GenericReady" and intent.get("desiredState") == "Running" + links = sandbox_ui_links(user, entry.get("profiles", []), catalog, domain) if catalog and entry.get("desiredState") != "Stopped" else [] + profiles = entry.get("profiles", []) + declared = [sb for p in profiles for _, sb in enabled_catalog_sandboxes(catalog[p])] if ( + catalog and all(p in catalog for p in profiles)) else [] + headless = bool(declared) and all(sb.get("uiRoute") is False for sb in declared) + ready = bool(running and (headless or (links and domain)) and app_status.get("sync", {}).get("status") == "Synced" + and app_status.get("health", {}).get("status") == "Healthy" and all(ping(url) for _,url in links)) + observations = [(True, "Registration recorded"), (app_ok, "Argo application for this owner"), + (bound, "Intent for this registration and update"), + (running, "Profile sandboxes running"), (ready, "Workspace ready" if headless else "Owner chat routes available")] + stopped = bool(current and desired == "Stopped" and st.get("phase") == "Stopped") + if desired == "Stopped": + observations[-2:] = [(stopped, "OpenShell sandboxes stopped"), (stopped, "Storage retained")] + phase = "none" + steps = [] + waiting = False + for stage, (done, title) in zip(STAGE_IDS, observations): + state = "waiting" if waiting else "done" if done else "active" + if state == "done": + phase = stage + else: + waiting = True + steps.append(dict(id=stage, title=title, detail=title, state=state)) + failed = bool(current and st.get("phase") == "Blocked" or app_ok and app_status.get("operationState", {}).get("phase") in ("Error", "Failed")) + if failed: + reason = st.get("reason") + if isinstance(reason, str) and reason in SAFE_FAILURE_REASONS: + message = f"Container reconciliation failed ({reason})" + elif (reason == "CoreRpcRejected" and isinstance(st.get("rpcCode"), str) + and st["rpcCode"] in SAFE_RPC_CODES): + message = f"Container reconciliation failed (OpenShell {st['rpcCode']})" + else: + message = "Container reconciliation failed" + else: + message = next((s["title"] for s in steps if s["state"] == "active"), "Workspace ready") + if stopped and not failed: + message = "Workspace stopped" + phase = "stopped" + return dict(user=user, deploymentMode=entry.get("values", {}).get("deployment", {}).get("mode", "container"), phase="failed" if failed else phase, title=message, + registration=reg, generation=rev, observedGeneration=st.get("observedGeneration"), + desiredState=desired, modelInputs=entry.get("modelInputs", {}), + message=message, text=message, ready=ready and not failed, failed=failed, steps=steps, + links=[dict(title=title,url=url) for title,url in links] if ready and not failed else []) + + + +def owner_route_available(url, issuer, client, timeout=3): + """Require an anonymous challenge AND a registered callback at the trusted issuer.""" + origin = url.rstrip("/") + auth = issuer.rstrip("/") + "/protocol/openid-connect/auth" + if not issuer.startswith("https://") or not origin.startswith("https://"): + return False + location = origin + for hop in range(3): + try: + with client.open(location, timeout=timeout) as res: + return (location.split("?")[0] == auth and res.status == 200 + and b'id="kc-form-login"' in res.read(256 * 1024)) + except urllib.error.HTTPError as exc: + with exc: + target = urllib.parse.urljoin(location, exc.headers.get("Location", "")) + if exc.code not in (302, 303, 307): + return False + parts = urllib.parse.urlsplit(target) + if hop == 0 and target.split("?")[0] == origin + "/oauth2/start": + location = target + continue + params = urllib.parse.parse_qs(parts.query) + if (target.split("?")[0] != auth or params.get("client_id") != ["openshell-dashboard"] + or params.get("redirect_uri") != [origin + "/oauth2/callback"]): + return False + location = target + except (OSError, ValueError): + return False + return False diff --git a/charts/saw-users/templates/_governed-proxy-policy.tpl b/charts/saw-users/templates/_governed-proxy-policy.tpl new file mode 100644 index 000000000..39b52d032 --- /dev/null +++ b/charts/saw-users/templates/_governed-proxy-policy.tpl @@ -0,0 +1,52 @@ +{{/* Shared broker config for VM and container. The catalog generator mirrors this template into saw-users. */}} +{{- define "saw.governedProxyPolicy" -}} +{{- $profiles := dict -}} +{{- $refresh := dict -}} +{{- range $profile := .catalog -}} + {{- range $credential := $profile.credentials | default list -}} + {{- $spec := $credential.refresh | default dict -}} + {{- if eq ($spec.strategy | default "") "oauth2_refresh_token" -}} + {{- $keys := list -}} + {{- range $spec.material -}}{{- $keys = append $keys .name -}}{{- end -}} + {{- range list "client_id" "client_secret" "refresh_token" -}} + {{- if not (has . $keys) -}}{{- fail "OAuth refresh requires client_id, client_secret and refresh_token material" -}}{{- end -}} + {{- end -}} + {{- $_ := set $refresh $profile.id (dict "credential" $credential.name "spec" (dict "strategy" $spec.strategy "tokenURL" $spec.token_url "refreshBeforeSeconds" $spec.refresh_before_seconds "maxLifetimeSeconds" $spec.max_lifetime_seconds "material" (dict "clientID" "client_id" "clientSecret" "client_secret" "refreshToken" "refresh_token"))) -}} + {{- end -}} + {{- end -}} + {{- if $profile.credentials -}} + {{- $hosts := list -}} + {{- range $profile.endpoints -}} + {{- if and .host (not (has .host $hosts)) -}}{{- $hosts = append $hosts .host -}}{{- end -}} + {{- end -}} + {{- $_ := set $profiles $profile.id $hosts -}} + {{- end -}} +{{- end -}} +{{- if hasKey $profiles "nvidia" -}}{{- $_ := set $profiles "build" (index $profiles "nvidia") -}}{{- end -}} +{{- $bindings := list -}} +{{- $attached := .attached -}} +{{- range .bindings -}} + {{- if has .name $attached -}} + {{- if and .name (not (regexMatch "^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$" .name)) -}}{{ fail (printf "invalid secret name %q" .name) }}{{- end -}} + {{- if not (regexMatch "^[A-Za-z0-9_.-]+$" (.key | default "")) -}} + {{- fail (printf "credentialProxy.secrets[%s].key is invalid" .name) -}} + {{- end -}} + {{- if not .providers -}}{{- fail (printf "credentialProxy.secrets[%s].providers is required" .name) -}}{{- end -}} + {{- if and (hasKey . "refresh") (not (kindIs "bool" .refresh)) -}} + {{- fail "credentialProxy.secrets[].refresh must be a boolean" -}} + {{- end -}} + {{- $binding := dict "secret" .name "key" .key "providers" .providers -}} + {{- if .refresh -}} + {{- if ne (len .providers) 1 -}}{{- fail "refresh binding requires exactly one provider type" -}}{{- end -}} + {{- $type := first .providers -}} + {{- if not (hasKey $refresh $type) -}}{{- fail (printf "provider %s has no OAuth refresh spec" $type) -}}{{- end -}} + {{- $entry := index $refresh $type -}} + {{- if ne .key $entry.credential -}}{{- fail "refresh binding key must match the profile credential" -}}{{- end -}} + {{- $_ := set $binding "refresh" $entry.spec -}} + {{- end -}} + {{- $bindings = append $bindings $binding -}} + {{- end -}} +{{- end -}} +{{- $policy := dict "markerPrefix" "saw-cred" "allowHosts" .allowHosts "profiles" $profiles "bindings" $bindings -}} +{{ $policy | toJson }} +{{- end -}} diff --git a/charts/saw-users/templates/_helpers.tpl b/charts/saw-users/templates/_helpers.tpl index 9344808a4..b3a987fa3 100644 --- a/charts/saw-users/templates/_helpers.tpl +++ b/charts/saw-users/templates/_helpers.tpl @@ -14,10 +14,17 @@ watches, not in whatever namespace this chart happens to be released into. {{- if not .Values.global.targetRevision -}} {{- fail "global.targetRevision is required" -}} {{- end -}} +{{- $revisions := .Values.workspaceRevisions -}} +{{- if not (kindIs "map" $revisions) -}}{{- fail "workspaceRevisions requires owner names and full Git commit SHAs" -}}{{- end -}} +{{- range $name, $revision := $revisions -}} +{{- if or (not (regexMatch "^[a-z0-9]([-a-z0-9]{0,17}[a-z0-9])?$" $name)) (not (kindIs "string" $revision)) (not (regexMatch "^[a-f0-9]{40}$" (toString $revision))) -}} +{{- fail "workspaceRevisions requires owner names and full Git commit SHAs" -}} +{{- end -}}{{- end -}} {{- $seen := dict -}} {{- range .Values.users | default list -}} {{- $name := .name | default "" | toString -}} -{{- if $.Values.ownerResolution.enabled -}} +{{- include "saw-users.validateDeploymentMode" (dict "root" $ "user" .) -}} +{{- if and $.Values.ownerResolution.enabled (not (has (dig "deployment" "mode" "vm" (.values | default dict)) (list "container" "kata"))) -}} {{- $entryValues := .values | default dict -}} {{- $entryAccess := index $entryValues "accessControl" | default dict -}} {{- if or (hasKey . "ownerSubject") (hasKey $entryAccess "ownerSubject") (hasKey $entryAccess "owner") -}} @@ -403,7 +410,7 @@ spec: namespace: {{ printf "saw-%s" $user.name }} source: repoURL: {{ $root.Values.global.repoURL | quote }} - targetRevision: {{ $root.Values.global.targetRevision | quote }} + targetRevision: {{ index $root.Values.workspaceRevisions $user.name | default $root.Values.global.targetRevision | quote }} path: {{ .path }} helm: releaseName: {{ .release }} @@ -414,6 +421,10 @@ spec: # hand (e.g. while cleaning up an older install) are restored. automated: selfHeal: true + {{- if and (eq .path "charts/saw-portal-container") (eq (int ($user.lifecycleVersion | default 0)) 1) }} + # Retire superseded publication Jobs. Audit archives opt out of prune and delete. + prune: true + {{- end }} retry: limit: {{ $root.Values.argo.retryLimit }} # Server-side apply: client-side apply copies every object into its @@ -424,3 +435,42 @@ spec: syncOptions: - ServerSideApply=true {{- end -}} + +{{/* Check effective backend before emitting any per-user Argo applications. +Validate both layers' shape, then apply the same defaults/user precedence as +openshellValues. Keep omission as vm without changing existing rendered values. */}} +{{- define "saw-users.validateDeploymentMode" -}} +{{- $mode := "vm" -}} +{{- range $values := list (.root.Values.defaults.openshellSaw | default dict) (.user.values | default dict) -}} +{{- if hasKey $values "deployment" -}} +{{- $deployment := get $values "deployment" -}} +{{- if not (kindIs "map" $deployment) -}} +{{- fail "deployment must be an object with a mode" -}} +{{- end -}} +{{- range $key, $_ := $deployment -}} +{{- if ne $key "mode" -}}{{- fail (printf "unknown deployment field %q" $key) -}}{{- end -}} +{{- end -}} +{{- if hasKey $deployment "mode" -}} +{{- $mode = get $deployment "mode" -}} +{{- if not (kindIs "string" $mode) -}}{{- fail "deployment.mode must be a string" -}}{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if not (has $mode (list "vm" "container" "kata")) -}} +{{- fail "deployment.mode must be one of vm, container, kata" -}} +{{- end -}} +{{- if and (has $mode (list "container" "kata")) (hasKey (.root.Values.containerSites | default dict) .user.name) -}} +{{- $u := .user -}} +{{- $site := index .root.Values.containerSites $u.name -}} +{{- if not (has (dig "deployment" "mode" "vm" ($u.values | default dict)) (list "container" "kata")) -}}{{- fail "deployment.mode=container must be explicit on each user" -}}{{- end -}} +{{- $foundation := eq (dig "automaticEnrollment" "enabled" true $site | toString) "false" -}} +{{- $catalog := (.root.Files.Get "files/profile-catalog.json" | fromJson).profiles -}} +{{- $profile := first ($u.profiles | default list) | default "" -}} +{{- $approved := or $foundation (hasKey (dig "automaticEnrollment" "approvedProfiles" dict $site) $profile) -}} +{{- if or (not $u.ownerSubject) (ne $u.ownerSubject $site.subject) (not (regexMatch "^[0-9a-f]{16}$" (default "" $u.registration))) (ne (len ($u.profiles | default list)) 1) (not (hasKey $catalog $profile)) (not $approved) (not $u.pruneOnRemove) -}} +{{- fail "container requires verified owner subject, scoped registration, approved BOM profile and foreground cleanup" -}} +{{- end -}} +{{- else if has $mode (list "container" "kata") -}} +{{- fail (printf "user %q: deployment.mode=container requires a containerSites entry" .user.name) -}} +{{- end -}} +{{- end -}} diff --git a/charts/saw-users/templates/applications.yaml b/charts/saw-users/templates/applications.yaml index c5460fb77..96905913b 100644 --- a/charts/saw-users/templates/applications.yaml +++ b/charts/saw-users/templates/applications.yaml @@ -1,5 +1,8 @@ {{- include "saw-users.validate" . -}} {{- range $user := .Values.users | default list }} +{{- if not (and $.Values.projection.enabled (has (dig "deployment" "mode" "vm" ($user.values | default dict)) (list "container" "kata"))) }} +{{- $container := has (dig "deployment" "mode" "vm" ($user.values | default dict)) (list "container" "kata") }} +{{- if not $container }} --- {{ include "saw-users.application" (dict "root" $ @@ -10,6 +13,7 @@ "release" (printf "saw-%s-secrets" $user.name) "values" (include "saw-users.vaultPrefix" (dict "root" $ "user" $user)) ) }} +{{- end }} --- {{ include "saw-users.application" (dict "root" $ @@ -21,6 +25,18 @@ "values" (include "saw-users.profiles" (dict "root" $ "user" $user)) ) }} --- +{{- if $container }} +{{- $site := deepCopy (index $.Values.containerSites $user.name | default dict) }} +{{- $mode := dig "deployment" "mode" "vm" ($user.values | default dict) }} +{{- if ne (eq $mode "kata") (not (empty $site.kataRuntimeClass)) }}{{ fail "registration placement differs from the site RuntimeClass approval" }}{{- end }} +{{- with $site.providerAllowHosts }} +{{- $scope := dict "root" $ "user" $user }} +{{- $policy := include "saw.governedProxyPolicy" (dict "catalog" ($.Files.Get "files/governed-profiles.json" | fromJson) "bindings" (include "saw-users.credentialBindings" $scope | fromJsonArray) "attached" (include "saw-users.secretNames" $scope | fromJsonArray) "allowHosts" .) | fromJson }} +{{- $_ := set $site "providerPolicy" $policy }} +{{- end }} +{{ $values := dict "owner" $user.name "ownerSubject" $user.ownerSubject "registration" $user.registration "vaultPrefix" $user.vaultPrefix "installationID" $user.installationID "intentGeneration" $user.intentGeneration "site" $site "global" $.Values.global }} +{{ include "saw-users.application" (dict "root" $ "user" $user "appName" (printf "saw-%s" $user.name) "wave" $.Values.syncWave.sandbox "path" "charts/saw-portal-container" "release" $user.name "values" (toYaml $values)) }} +{{- else }} {{ include "saw-users.application" (dict "root" $ "user" $user @@ -32,3 +48,5 @@ "values" (include "saw-users.openshellValues" (dict "root" $ "user" $user)) ) }} {{- end }} +{{- end }} +{{- end }} diff --git a/charts/saw-users/templates/namespaces.yaml b/charts/saw-users/templates/namespaces.yaml index ee63dcb0d..dce93a39f 100644 --- a/charts/saw-users/templates/namespaces.yaml +++ b/charts/saw-users/templates/namespaces.yaml @@ -1,12 +1,19 @@ {{- include "saw-users.validate" . -}} {{- range $user := .Values.users | default list }} +{{- if not (and $.Values.projection.enabled (has (dig "deployment" "mode" "vm" ($user.values | default dict)) (list "container" "kata"))) }} --- apiVersion: v1 kind: Namespace metadata: name: {{ printf "saw-%s" $user.name }} labels: + {{- if has (dig "deployment" "mode" "vm" ($user.values | default dict)) (list "container" "kata") }} + pod-security.kubernetes.io/enforce: restricted + saw.redhat.com/runtime-owner: {{ required "installationID required" $user.installationID | quote }} + saw.redhat.com/registration: {{ $user.registration | quote }} + {{- else }} openshell.pattern/saw: "true" + {{- end }} openshell.pattern/owner: {{ $user.name | quote }} argocd.argoproj.io/managed-by: {{ include "saw-users.argoNamespace" $ | quote }} {{- range $k, $v := $.Values.namespaceLabels | default dict }} @@ -18,3 +25,4 @@ metadata: argocd.argoproj.io/sync-options: Prune=false {{- end }} {{- end }} +{{- end }} diff --git a/charts/saw-users/templates/owner-resolution.yaml b/charts/saw-users/templates/owner-resolution.yaml index 619354d5e..d3957601b 100644 --- a/charts/saw-users/templates/owner-resolution.yaml +++ b/charts/saw-users/templates/owner-resolution.yaml @@ -1,6 +1,7 @@ {{- if .Values.ownerResolution.enabled -}} {{- include "saw-users.validate" . -}} {{- range $user := .Values.users | default list }} +{{- if not (has (dig "deployment" "mode" "vm" ($user.values | default dict)) (list "container" "kata")) }} --- apiVersion: v1 kind: ConfigMap @@ -36,3 +37,4 @@ subjects: namespace: {{ $.Values.ownerResolution.resolverNamespace }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/saw-users/templates/projector.yaml b/charts/saw-users/templates/projector.yaml new file mode 100644 index 000000000..0ef7fd960 --- /dev/null +++ b/charts/saw-users/templates/projector.yaml @@ -0,0 +1,312 @@ +{{- if .Values.projection.enabled }} +{{- $users := list }} +{{- range $user := .Values.users }} +{{- if has (dig "deployment" "mode" "vm" ($user.values | default dict)) (list "container" "kata") }} +{{- $users = append $users $user }} +{{- end }}{{- end }} +{{- $owner := printf "%s/%s" .Release.Namespace .Release.Name | sha256sum | trunc 20 }} +{{- $name := printf "saw-projector-%s" $owner }} +{{- $config := dict }} +{{- if $users }}{{- $config = deepCopy .Values }}{{- end }} +{{- $_ := set $config "users" $users }} +{{- range $user := $users }} +{{- $site := index $config.containerSites $user.name }} +{{- if not $site }}{{ fail (printf "user %q: deployment.mode=container requires a containerSites entry" $user.name) }}{{ end }} +{{- $_ := set $site "observer" (dict "namespace" $.Release.Namespace "serviceAccount" $name) }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $name }}-configuration} +data: + workspaces.json: {{ $config | toJson | quote }} +{{- if $users }} +{{- $argo := include "saw-users.argoNamespace" . }} +{{- if eq .Release.Namespace $argo }}{{ fail "workspace projection requires a dedicated namespace outside Argo CD" }}{{ end }} +{{- if or (not .Values.projection.egress) (not (regexMatch "^.+@sha256:[a-f0-9]{64}$" .Values.projection.image)) }}{{ fail "projection requires a pinned helper image and explicit API/DNS egress" }}{{ end }} +{{- if not .Values.projection.applicationSetControllerLabels }}{{ fail "projection requires the actual ApplicationSet controller Pod labels" }}{{ end }} +{{- $identity := dict "namespace" .Release.Namespace "argoNamespace" $argo "name" $name "owner" $owner }} +{{- $identityCode := .Files.Get "files/projector_identity.py" }} +{{- $bootstrapRevision := dict "config" $identity "source" $identityCode "image" .Values.projection.image | toJson | sha256sum | trunc 12 }} +{{- $bootstrap := printf "%s-%s" $name $bootstrapRevision }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $name }}} +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $name }}-identity} +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $name }}-source} +data: + projector.py: | +{{ .Files.Get "files/projector.py" | indent 4 }} + workspace.py: | +{{ .Files.Get "files/workspace.py" | indent 4 }} + preflight.py: | +{{ .Files.Get "files/preflight.py" | indent 4 }} + profile-catalog.json: | +{{ .Files.Get "files/profile-catalog.json" | indent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $bootstrap }}} +immutable: true +data: + identity.json: {{ $identity | toJson | quote }} + projector_identity.py: | +{{ $identityCode | indent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $name }}, namespace: {{ $argo }}} +rules: +- apiGroups: [argoproj.io] + resources: [applications] + verbs: [get] + resourceNames: +{{- range $user := $users }} + - {{ $name }}-{{ $user.name }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $name }}, namespace: {{ $argo }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }}} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ .Release.Namespace }}}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: {name: {{ $name }}-identity} +rules: [{apiGroups: [""], resources: [namespaces], resourceNames: [{{ .Release.Namespace }}, {{ $argo }}], verbs: [get]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: {name: {{ $name }}-identity} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ $name }}-identity} +subjects: [{kind: ServiceAccount, name: {{ $name }}-identity, namespace: {{ .Release.Namespace }}}] +--- +{{- $classes := list }} +{{- range $user := $users }} +{{- if eq (dig "deployment" "mode" "vm" ($user.values | default dict)) "kata" }} +{{- $classes = append $classes (dig "kataRuntimeClass" "name" "" (index $.Values.containerSites $user.name)) }} +{{- end }} +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: {name: {{ $name }}-preflight} +rules: +- {apiGroups: [""], resources: [namespaces], resourceNames: [{{ range $user := $users }}{{ printf "saw-%s" $user.name | quote }},{{ end }}], verbs: [get]} +- {apiGroups: [admissionregistration.k8s.io], resources: [validatingadmissionpolicies, validatingadmissionpolicybindings], resourceNames: [{{ range $user := $users }}{{ printf "saw-saw-%s-runtime" $user.name | quote }},{{ printf "saw-saw-%s-runtime-kata" $user.name | quote }},{{ end }}], verbs: [get]} +{{- if $classes }} +- {apiGroups: [node.k8s.io], resources: [runtimeclasses], resourceNames: {{ $classes | uniq | toJson }}, verbs: [get]} +- {apiGroups: [""], resources: [nodes], verbs: [list]} +- {apiGroups: [config.openshift.io], resources: [clusterversions], resourceNames: [version], verbs: [get]} +- {apiGroups: [kataconfiguration.openshift.io], resources: [kataconfigs], verbs: [list]} +- {apiGroups: [apiextensions.k8s.io], resources: [customresourcedefinitions], resourceNames: [kataconfigs.kataconfiguration.openshift.io, sandboxes.agents.x-k8s.io, externalsecrets.external-secrets.io, adminnetworkpolicies.policy.networking.k8s.io], verbs: [get]} +- {nonResourceURLs: [/apis/admissionregistration.k8s.io/v1], verbs: [get]} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: {name: {{ $name }}-preflight} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: {{ $name }}-preflight} +subjects: [{kind: ServiceAccount, name: {{ $name }}, namespace: {{ .Release.Namespace }}}] +--- +# Kubernetes RBAC cannot limit CREATE by resourceNames. Fence the bootstrap +# identity before granting it create in either namespace, especially Argo CD. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ $name }}-credentials + annotations: {argocd.argoproj.io/sync-wave: "-12"} +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - {apiGroups: [""], apiVersions: [v1], resources: [secrets], operations: [CREATE]} + matchConditions: + - name: projector-bootstrap + expression: request.userInfo.username == {{ printf "system:serviceaccount:%s:%s-identity" .Release.Namespace $name | toJson }} + validations: + - expression: >- + request.namespace in [{{ .Release.Namespace | toJson }}, {{ $argo | toJson }}] && + object.metadata.name == {{ $name | toJson }} && object.type == 'Opaque' && + has(object.immutable) && object.immutable && has(object.data) && + object.data.size() == 1 && 'token' in object.data && + has(object.metadata.labels) && object.metadata.labels.size() == 2 && + object.metadata.labels['saw.redhat.com/projector-owner'] == {{ $owner | toJson }} && + object.metadata.labels['app.kubernetes.io/part-of'] == 'argocd' && + has(object.metadata.annotations) && object.metadata.annotations.size() == 1 && + object.metadata.annotations['saw.redhat.com/identity-sha256'].matches('^[a-f0-9]{64}$') && + has(object.metadata.ownerReferences) && object.metadata.ownerReferences.size() == 1 && + object.metadata.ownerReferences[0].name == {{ $name | toJson }} && + (request.namespace == {{ $argo | toJson }} ? + (object.metadata.ownerReferences[0].kind == 'ApplicationSet' && object.metadata.ownerReferences[0].apiVersion == 'argoproj.io/v1alpha1') : + (object.metadata.ownerReferences[0].kind == 'Deployment' && object.metadata.ownerReferences[0].apiVersion == 'apps/v1')) + message: Only the named immutable Opaque plugin credential may be created. +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ $name }}-credentials + annotations: {argocd.argoproj.io/sync-wave: "-11"} +spec: {policyName: {{ $name }}-credentials, validationActions: [Deny]} +{{- range $ns := list .Release.Namespace $argo }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $name }}-identity, namespace: {{ $ns }}} +rules: +- {apiGroups: [""], resources: [secrets], resourceNames: [{{ $name }}], verbs: [get]} +# Create is namespace-scoped in Kubernetes RBAC; the short-lived bootstrap is trusted. +- {apiGroups: [""], resources: [secrets], verbs: [create]} +{{- if eq $ns $.Release.Namespace }} +- {apiGroups: [apps], resources: [deployments], resourceNames: [{{ $name }}], verbs: [get]} +{{- else }} +- {apiGroups: [argoproj.io], resources: [applicationsets], resourceNames: [{{ $name }}], verbs: [get]} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $name }}-identity, namespace: {{ $ns }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }}-identity} +subjects: [{kind: ServiceAccount, name: {{ $name }}-identity, namespace: {{ $.Release.Namespace }}}] +{{- end }} +--- +apiVersion: batch/v1 +kind: Job +metadata: {name: {{ $bootstrap }}} +spec: + backoffLimit: 3 + activeDeadlineSeconds: 300 + template: + metadata: {labels: {app.kubernetes.io/name: {{ $name }}}} + spec: + serviceAccountName: {{ $name }}-identity + automountServiceAccountToken: true + restartPolicy: Never + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: initialize + image: {{ .Values.projection.image }} + command: [python3, /source/projector_identity.py] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + env: [{name: PYTHONDONTWRITEBYTECODE, value: '1'}] + resources: {requests: {cpu: 10m, memory: 32Mi}, limits: {memory: 128Mi}} + volumeMounts: [{name: source, mountPath: /source, readOnly: true}] + volumes: [{name: source, configMap: {name: {{ $bootstrap }}}}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + labels: {saw.redhat.com/projector-owner: {{ $owner | quote }}} +spec: + replicas: 1 + selector: {matchLabels: {app.kubernetes.io/name: {{ $name }}, app.kubernetes.io/component: projector}} + template: + metadata: + labels: {app.kubernetes.io/name: {{ $name }}, app.kubernetes.io/component: projector} + annotations: + checksum/source: {{ printf "%s%s%s%s" (.Files.Get "files/projector.py") (.Files.Get "files/workspace.py") (.Files.Get "files/preflight.py") (.Files.Get "files/profile-catalog.json") | sha256sum }} + spec: + serviceAccountName: {{ $name }} + automountServiceAccountToken: true + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: projector + image: {{ .Values.projection.image }} + command: [python3, /source/projector.py] + env: + - {name: PROJECTOR_NAME, value: {{ $name }}} + - {name: ARGO_NAMESPACE, value: {{ $argo }}} + - {name: PYTHONDONTWRITEBYTECODE, value: '1'} + ports: [{name: http, containerPort: 4355}] + readinessProbe: {httpGet: {path: /healthz, port: http}} + resources: {requests: {cpu: 10m, memory: 64Mi}, limits: {memory: 128Mi}} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + volumeMounts: + - {name: source, mountPath: /source, readOnly: true} + - {name: configuration, mountPath: /config, readOnly: true} + - {name: identity, mountPath: /identity, readOnly: true} + volumes: + - {name: source, configMap: {name: {{ $name }}-source}} + - {name: configuration, configMap: {name: {{ $name }}-configuration}} + - {name: identity, secret: {secretName: {{ $name }}}} +--- +apiVersion: v1 +kind: Service +metadata: {name: {{ $name }}} +spec: + selector: {app.kubernetes.io/name: {{ $name }}, app.kubernetes.io/component: projector} + ports: [{name: http, port: 4355, targetPort: http}] +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $name }}} +spec: + podSelector: {matchLabels: {app.kubernetes.io/name: {{ $name }}}} + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $argo }}}} + podSelector: {matchLabels: {{ .Values.projection.applicationSetControllerLabels | toJson }}} + ports: [{protocol: TCP, port: 4355}] + egress: {{ .Values.projection.egress | toJson }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $name }}, namespace: {{ $argo }}} +data: + token: {{ printf "$%s:token" $name | quote }} + baseUrl: http://{{ $name }}.{{ .Release.Namespace }}.svc:4355 +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: {{ $name }} + namespace: {{ $argo }} + labels: {saw.redhat.com/projector-owner: {{ $owner | quote }}} +spec: + goTemplate: true + goTemplateOptions: [missingkey=error] + syncPolicy: + applicationsSync: sync + preserveResourcesOnDeletion: false + generators: + - plugin: + configMapRef: {name: {{ $name }}} + requeueAfterSeconds: 30 + template: + metadata: + name: '{{ $name }}-{{ "{{" }} .name {{ "}}" }}' + labels: {saw.redhat.com/projector-owner: {{ $owner | quote }}, openshell.pattern/owner: '{{ "{{" }} .name {{ "}}" }}'} + finalizers: [resources-finalizer.argocd.argoproj.io/foreground] + spec: + project: {{ .Values.argo.project }} + destination: {server: 'https://kubernetes.default.svc', namespace: {{ .Release.Namespace }}} + source: + repoURL: {{ .Values.global.repoURL | quote }} + targetRevision: '{{ "{{" }} .revision {{ "}}" }}' + path: charts/saw-users + helm: + releaseName: 'workspace-{{ "{{" }} .name {{ "}}" }}' + values: '{{ "{{" }} .workspaceValues {{ "}}" }}' + # Argo owns these two hook-state finalizers. Preserve them during sync; + # keep detecting changes to the resource-deletion finalizer and app spec. + ignoreDifferences: + - group: argoproj.io + kind: Application + jqPathExpressions: + - '.metadata.finalizers[]? | select(. == "pre-delete-finalizer.argocd.argoproj.io" or . == "pre-delete-finalizer.argocd.argoproj.io/cleanup")' + syncPolicy: + automated: {prune: true, selfHeal: true} + retry: {limit: 20, refresh: true} + syncOptions: [PrunePropagationPolicy=foreground, RespectIgnoreDifferences=true] +{{- end }}{{- end }} diff --git a/charts/saw-users/values.yaml b/charts/saw-users/values.yaml index f75af2b86..c863a7f3c 100644 --- a/charts/saw-users/values.yaml +++ b/charts/saw-users/values.yaml @@ -7,6 +7,11 @@ # Set it to false when that user's Vault prefix has no web-search key. users: [] +# Operator-owned revision pins for individual owners, in global.repoURL. +# Each value is a full Git commit SHA; unlisted owners use global.targetRevision. +# RHDH passes the same map through sawUsers.workspaceRevisions. +workspaceRevisions: {} + # Pattern Keycloak accounts are resolved centrally. Disable this only for an # external issuer and supply a verified ownerSubject in each user entry. ownerResolution: @@ -132,3 +137,14 @@ global: clusterDomain: example.com pattern: secure-agent-workspace vpArgoNamespace: vp-gitops + +# Static container entries use the same workspace projector as the portal. +# RHDH supplies already-projected values and leaves this disabled. +projection: + enabled: false + image: registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c + # Explicit API/DNS egress for the projector and its identity bootstrap. + egress: [] + # Match the selected OpenShift GitOps instance's controller Pod, not a + # presumed instance name. Used together with the Argo namespace selector. + applicationSetControllerLabels: {} diff --git a/charts/saw-workspace-relay/Chart.yaml b/charts/saw-workspace-relay/Chart.yaml new file mode 100644 index 000000000..d6b0f0288 --- /dev/null +++ b/charts/saw-workspace-relay/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: saw-workspace-relay +description: Workspace-scoped Keycloak service account and owner-authenticated OpenShell UI forward +version: 0.1.0 diff --git a/charts/saw-workspace-relay/files/bind.py b/charts/saw-workspace-relay/files/bind.py new file mode 100644 index 000000000..385e88444 --- /dev/null +++ b/charts/saw-workspace-relay/files/bind.py @@ -0,0 +1,86 @@ +"""Grant the relay the stock workspace user role after checking the live owner and IDs.""" +import json +import os +import uuid +import time +from pathlib import Path +from saw_kubernetes.reconcile import RpcError +from saw_kubernetes.admin_session import GatewaySessions, TrustBundle +from saw_kubernetes.controller import KubernetesClient + + +def main(): + ws, wid = os.environ["WORKSPACE"], os.environ["WORKSPACE_ID"] + relay, owner = os.environ["SUBJECT"], os.environ["OWNER"] + if (any(uuid.UUID(v).int == 0 for v in (relay, owner, os.environ["INSTALLATION"])) + or relay == owner): + raise ValueError("invalid relay scope") + service_account = Path("/var/run/secrets/kubernetes.io/serviceaccount") + api = KubernetesClient("https://kubernetes.default.svc", service_account/"ca.crt", service_account/"token") + trust = TrustBundle(api.request, os.environ["GATEWAY_NAMESPACE"], os.environ["TRUSTED_NAMESPACE"], + os.environ["GATEWAY"], os.environ["INSTALLATION"], Path("/identity/ca.crt").read_bytes(), + os.environ["POD_NAME"], os.environ["POD_UID"]) + with GatewaySessions(trust, os.environ["ENDPOINT"]).open() as g: + bind(g, ws, wid, relay, owner, os.environ["INSTALLATION"], os.environ.get("DISCOVER_ID") == "true") + + +def bind(gateway, workspace, expected_id, relay, owner, installation, discover, *, + timeout=210, monotonic=time.monotonic, sleep=time.sleep): + scope = {"workspace_scope": {"workspace": workspace}} + + def members(): + roles, seen = {}, set() + page = "" + while True: + response = gateway.call("ListWorkspaceMembers", {**scope, "page_size": 100, "page_token": page}) + for member in response.get("members", []): + roles[member["principal_subject"]] = member["role"] + page = response.get("next_page_token", "") + if not page: + return roles + if page in seen or len(seen) >= 100: + raise ValueError("invalid membership pagination") + seen.add(page) + + # The reconciler creates the workspace and assigns its owner in separate passes. + # Pin the first accepted UUID across observations before granting any access. + deadline = monotonic() + timeout + while True: + try: + meta = gateway.call("GetWorkspace", {"name": workspace})["workspace"]["metadata"] + labels = meta.get("labels", {}) + builtin = discover and workspace == "default" and not labels + managed = (labels.get("saw.redhat.com/managed-by") == "kubernetes-runtime" + and labels.get("saw.redhat.com/runtime-owner") == installation) + if not expected_id and discover: + expected_id = meta["id"] + if (not expected_id or not uuid.UUID(expected_id).int or meta["id"] != expected_id + or meta["name"] != workspace or meta.get("deletion_time") or not (managed or builtin)): + raise ValueError("workspace identity mismatch") + roles = members() + if builtin and set(roles) - {owner, relay}: + raise ValueError("unexpected membership in dedicated default workspace") + if relay in roles and roles[relay] != "WORKSPACE_ROLE_USER": + raise ValueError("unexpected relay role") + if owner in roles: + if roles[owner] != "WORKSPACE_ROLE_ADMIN": + raise ValueError("owner mismatch") + break + except RpcError as exc: + if exc.code not in {"NOT_FOUND", "UNAVAILABLE"}: + raise + remaining = deadline - monotonic() + if remaining <= 0: + raise TimeoutError("verified workspace and owner membership were not ready") + sleep(min(2, remaining)) + + if relay not in roles: + gateway.call("AddWorkspaceMember", {**scope, "principal_subject": relay, "role": "WORKSPACE_ROLE_USER", + "request_id": str(uuid.uuid5(uuid.UUID(expected_id), "relay-user/" + relay))}) + if members().get(relay) != "WORKSPACE_ROLE_USER": + raise ValueError("unexpected relay role") + print(json.dumps({"workspace": workspace, "workspaceID": expected_id, "subject": relay, "role": "WORKSPACE_ROLE_USER"})) + + +if __name__ == "__main__": + main() diff --git a/charts/saw-workspace-relay/files/identity.py b/charts/saw-workspace-relay/files/identity.py new file mode 100644 index 000000000..39c166513 --- /dev/null +++ b/charts/saw-workspace-relay/files/identity.py @@ -0,0 +1,180 @@ +"""Bootstrap one dedicated confidential client. Never log HTTP bodies or credentials.""" +import base64 +import json +import os +import re +from pathlib import Path +import secrets +import ssl +import sys +from urllib.parse import urlencode, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener, HTTPSHandler +from urllib.error import HTTPError + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args): + return None + + +def api(url, *, ca=None, method="GET", token=None, body=None, form=None, patch=False): + headers = {} + if urlsplit(url).scheme != "https": + raise RuntimeError("HTTPS is required") + if token: + headers["Authorization"] = "Bearer " + token + data = None + if form is not None: + data = urlencode(form).encode() + headers["Content-Type"] = "application/x-www-form-urlencoded" + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/merge-patch+json" if patch else "application/json" + client = build_opener(NoRedirect(), HTTPSHandler(context=ssl.create_default_context(cafile=ca))) + with client.open(Request(url, data, headers, method=method), timeout=15) as response: + raw = response.read() + return json.loads(raw) if raw else None + + +def claims(token): + part = token.split(".")[1] + return json.loads(base64.urlsafe_b64decode(part + "=" * (-len(part) % 4))) + + +def main(): + issuer = os.environ["ISSUER"].rstrip("/") + base, realm = issuer.rsplit("/realms/", 1) + cid = os.environ["CLIENT_ID"] + credential = Path("/kc-provisioner") + if not realm or realm == "master": + raise RuntimeError("a dedicated workspace realm is required") + deleting = os.environ.get("DELETE_BOUND_CLIENT") == "true" + if not deleting and (credential/"issuer").read_text().strip().rstrip("/") != issuer: + raise RuntimeError("provisioning credential belongs to a different issuer") + marker = os.environ["INSTALLATION"] + "/" + os.environ["WORKSPACE"] + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", cid): + raise RuntimeError("expected a registration-scoped relay client ID") + registration_url = issuer + "/clients-registrations/default" + sa = Path("/var/run/secrets/kubernetes.io/serviceaccount") + address = "https://kubernetes.default.svc/api/v1/namespaces/" + os.environ["UI_NAMESPACE"] + "/secrets/relay-credentials" + def secret(method="GET", body=None): + return api(address, method=method, token=(sa/"token").read_text(), ca=str(sa/"ca.crt"), body=body, patch=bool(body)) + opts = dict(token=(sa/"token").read_text(), ca=str(sa/"ca.crt")) + state_url = ("https://kubernetes.default.svc/api/v1/namespaces/" + os.environ["PROVISIONER_NAMESPACE"] + + "/secrets/" + os.environ["REGISTRATION_STATE"]) + state = api(state_url, **opts) + if state["metadata"]["labels"].get("saw.redhat.com/runtime-owner") != os.environ["INSTALLATION"]: + raise RuntimeError("foreign client registration state") + raw = state.get("data", {}).get("registration-token") + registration_token = base64.b64decode(raw, validate=True).decode() if raw else None + if registration_token and (state["data"].get("issuer") != base64.b64encode(issuer.encode()).decode() + or state["data"].get("attempt") != base64.b64encode(cid.encode()).decode()): + raise RuntimeError("client registration receipt belongs to a different issuer or client") + if deleting: + if cid != "saw-portal-" + os.environ["REGISTRATION"]: + raise RuntimeError("refusing foreign client deletion") + if not registration_token: + if state.get("data", {}).get("attempt"): + raise RuntimeError("ambiguous client create: restore its registration receipt") + return + try: + api(registration_url + "/" + cid, method="DELETE", token=registration_token) + except HTTPError as exc: + if exc.code != 404: + raise + print("registration-scoped relay client removed") + return + obj = secret() + if obj["metadata"]["labels"].get("saw.redhat.com/runtime-owner") != os.environ["INSTALLATION"]: + raise RuntimeError("foreign credential Secret") + if not obj.get("data"): + if registration_token: + raise RuntimeError("registered client without retained credential") + obj = secret("PATCH", {"metadata":{"resourceVersion":obj["metadata"]["resourceVersion"]}, + "stringData":{"client-secret":secrets.token_urlsafe(40), + "cookie-secret":base64.urlsafe_b64encode(secrets.token_bytes(32)).decode()}}) + if "relay-proof" not in obj["data"]: + obj = secret("PATCH", {"metadata":{"resourceVersion":obj["metadata"]["resourceVersion"]}, + "stringData":{"relay-proof":secrets.token_hex(32)}}) + proof = obj["data"]["relay-proof"] + if not re.fullmatch(r"[0-9a-f]{64}", base64.b64decode(proof, validate=True).decode()): + raise RuntimeError("invalid relay proof") + proof_url = ("https://kubernetes.default.svc/api/v1/namespaces/" + os.environ["RUNTIME_NAMESPACE"] + + "/secrets/" + os.environ["PROOF_NAME"]) + opts = dict(token=(sa/"token").read_text(), ca=str(sa/"ca.crt")) + target = api(proof_url, **opts) + if target["metadata"]["labels"].get("saw.redhat.com/runtime-owner") != os.environ["INSTALLATION"]: + raise RuntimeError("foreign proof projection") + if target.get("data") and target["data"] != {"relay-proof":proof}: + raise RuntimeError("relay proof projection differs") + api(proof_url, method="PATCH", patch=True, body={"metadata":{"resourceVersion":target["metadata"]["resourceVersion"]}, + "immutable":True, "data":{"relay-proof":proof}}, **opts) + password = base64.b64decode(obj["data"]["client-secret"]).decode() + if not registration_token: + # The registrar needs create-client only. An administrator-defined + # client scope supplies the audience and sole openshell-user role. + token = api(issuer + "/protocol/openid-connect/token", method="POST", form={ + "grant_type":"client_credentials", "client_id":(credential/"client-id").read_text().strip(), + "client_secret":(credential/"client-secret").read_text().strip()})["access_token"] + settings = dict(clientId=cid, protocol="openid-connect", clientAuthenticatorType="client-secret", + publicClient=False, enabled=True, serviceAccountsEnabled=True, standardFlowEnabled=False, + implicitFlowEnabled=False, directAccessGrantsEnabled=False, fullScopeAllowed=False, + secret=password, redirectUris=[], webOrigins=[], optionalClientScopes=[], + defaultClientScopes=["basic", "saw-workspace-relay"], attributes={"saw.scope":marker}) + if state.get("data", {}).get("attempt"): + raise RuntimeError("previous client creation may have committed; operator recovery required") + state = api(state_url, method="PATCH", patch=True, **opts, body={ + "metadata":{"resourceVersion":state["metadata"]["resourceVersion"]}, + "stringData":{"attempt":cid, "issuer":issuer}}) + try: + created = api(registration_url, method="POST", token=token, body=settings) + except HTTPError as exc: + if exc.code in (400, 401, 403): + api(state_url, method="PATCH", patch=True, **opts, body={ + "metadata":{"resourceVersion":state["metadata"]["resourceVersion"]}, + "data":{"attempt":None}}) + raise + registration_token = created.get("registrationAccessToken") + if (not registration_token or created.get("clientId") != cid + or created.get("attributes", {}).get("saw.scope") != marker): + raise RuntimeError("client registration receipt unavailable") + state = api(state_url, method="PATCH", patch=True, body={ + "metadata":{"resourceVersion":state["metadata"]["resourceVersion"]}, + "stringData":{"registration-token":registration_token, "client-secret":created.get("secret") or password}}, **opts) + password = base64.b64decode(state["data"]["client-secret"], validate=True).decode() + if obj["data"]["client-secret"] != state["data"]["client-secret"]: + obj = secret("PATCH", {"metadata":{"resourceVersion":obj["metadata"]["resourceVersion"]}, + "data":{"client-secret":state["data"]["client-secret"]}}) + access = api(issuer + "/protocol/openid-connect/token", method="POST", form={ + "grant_type":"client_credentials","client_id":cid,"client_secret":password})["access_token"] + c = claims(access) + sub = c["sub"] + if not re.fullmatch(r"[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}", sub): + raise RuntimeError("invalid relay subject") + aud = c["aud"] if isinstance(c["aud"],list) else [c["aud"]] + if (c["azp"] != cid or c["iss"] != issuer or set(aud) != {os.environ["AUDIENCE"]} + or set(c.get("realm_access",{}).get("roles",[])) != {"openshell-user"} or c.get("resource_access")): + raise RuntimeError("unexpected relay token scope") + if public_ns := os.environ.get("PUBLIC_SUBJECT_NAMESPACE"): + name=os.environ.get("PUBLIC_SUBJECT_MAP","saw-portal-relay-subject") + suffix=cid.removeprefix("saw-portal-") + permitted=(name=="saw-portal-relay-subject" and suffix==public_ns[8:]) or name=="saw-portal-relay-"+suffix + if not re.fullmatch("saw-ctl-[a-f0-9]{16}",public_ns) or not re.fullmatch("saw-portal-[a-f0-9]{16}",cid) or not permitted: + raise RuntimeError("invalid registration identity") + path="https://kubernetes.default.svc/api/v1/namespaces/"+public_ns+"/configmaps/"+name + opts=dict(token=(sa/"token").read_text(),ca=str(sa/"ca.crt")) + cm=api(path,**opts) + if cm["metadata"]["labels"].get("saw.redhat.com/runtime-owner")!=os.environ["INSTALLATION"] or cm.get("data",{}).get("subject",sub)!=sub: + raise RuntimeError("foreign public subject binding") + api(path,method="PATCH",patch=True,body={"metadata":{"resourceVersion":cm["metadata"]["resourceVersion"]},"immutable":True,"data":{"subject":sub}},**opts) + print(json.dumps({"clientID":cid,"subject":sub,"workspace":os.environ["WORKSPACE"], + "audience":aud,"roles":["openshell-user"]})) + + +if __name__ == "__main__": + try: + main() + except HTTPError as e: + sys.exit("Identity bootstrap HTTP " + str(e.code)) + except Exception as e: + sys.exit("Relay check failed: " + (str(e) if type(e) is RuntimeError else type(e).__name__)) diff --git a/charts/saw-workspace-relay/files/saw_ui_limit.py b/charts/saw-workspace-relay/files/saw_ui_limit.py new file mode 100644 index 000000000..c5c29d009 --- /dev/null +++ b/charts/saw-workspace-relay/files/saw_ui_limit.py @@ -0,0 +1,369 @@ +"""Caps the connections to `openshell forward service` (SAW sandbox UI). + +OpenShell allows 20 concurrent forward connections per sandbox and closes +the rest (RESOURCE_EXHAUSTED, "sandbox SSH connection limit reached", +NVIDIA/OpenShell#3494). A browser loading the OpenClaw control UI through +the route opens more than that at once, and the requests that lost came +back as 502/504 after 30 s. This TCP relay sits between oauth2-proxy and the +forward: + +- at most MAX connections reach the forward; the others wait for a free one; +- while some wait, an HTTP keep-alive connection that has been idle for + IDLE_PREEMPT seconds after an answer is closed to make room (an HTTP + client opens a new one); WebSocket connections are never closed; +- a connection the forward closes or resets before answering is retried; +- a GET that gets no answer at all within ANSWER_WAIT is sent again on new + connections, and the first answer wins. + + python3 saw_ui_limit.py LISTEN_PORT UPSTREAM_PORT [MAX] [--connect-only] + +The container relay uses --connect-only: reconnect only before sending bytes, +never replay an HTTP request or WebSocket frame. The stable listener absorbs +brief stock-CLI restarts. Established WebSockets still close when upstream +authentication expires; clients must reconnect normally. +""" +import asyncio +import os +import re +from pathlib import Path +import sys +import time + + +def log(message): + print(message, file=sys.stderr, flush=True) + +RETRIES = 8 +BUFFER_LIMIT = 1 << 20 +ANSWER_WAIT = 2.0 # seconds to wait for a first answer byte before giving up retries +IDLE_PREEMPT = 2.0 # seconds an answered keep-alive connection may hold a slot others wait for +HEDGES = 2 # extra connections for a repeatable request that gets no answer +HEDGE_TOTAL = 25.0 # seconds before giving up on such a request (the router allows 30) +# For a request that is not safe to send again and lost its answer. +_AMBIGUOUS_BODY = b"The sandbox closed the connection; the request may have been processed.\n" +AMBIGUOUS_ANSWER = (b"HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\n" + b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(_AMBIGUOUS_BODY)) + _AMBIGUOUS_BODY + + +class Conn: + """One client connection holding (or waiting for) a slot.""" + + def __init__(self, websocket): + self.websocket = websocket + self.last = time.monotonic() + self.answered = False # the forward spoke last: no request in flight + self.writers = [] + + def touch(self, answered): + self.last = time.monotonic() + self.answered = answered + + def idle_for(self): + return time.monotonic() - self.last + + def close(self): + for w in self.writers: + try: + w.transport.abort() + except Exception: + pass + + +class Slots: + """MAX upstream connections at a time; idle keep-alive ones give way. + + A client connection holds one slot for its upstream connection. A hedge + (an extra connection for a repeatable request that got no answer) takes + a slot of its own, only when one is free, and gives it back as soon as + it is closed or becomes the connection that answered.""" + + def __init__(self, limit): + self.limit = limit + self.active = set() + self.extra = 0 # hedge connections open now + self.cond = asyncio.Condition() + + def in_use(self): + return len(self.active) + self.extra + + def try_extra(self): + """Take a slot for a hedge connection if one is free; never waits. + (No await: atomic within the event loop.)""" + if self.in_use() >= self.limit: + return False + self.extra += 1 + return True + + async def release_extra(self, count=1): + if count <= 0: + return + async with self.cond: + self.extra = max(0, self.extra - count) + self.cond.notify(count) + + def victim(self): + idle = [c for c in self.active + if not c.websocket and c.answered and c.idle_for() >= IDLE_PREEMPT] + return min(idle, key=lambda c: c.last) if idle else None + + def describe(self): + ws = sum(1 for c in self.active if c.websocket) + busy = sum(1 for c in self.active if not c.websocket and not c.answered) + idle = sorted(round(c.idle_for(), 1) for c in self.active if not c.websocket and c.answered) + return (f"{self.in_use()}/{self.limit} in use: {ws} websocket, {busy} awaiting an answer, " + f"{len(idle)} answered (idle s: {idle}), {self.extra} hedge") + + async def acquire(self, conn): + start = time.monotonic() + logged = 0.0 + async with self.cond: + while self.in_use() >= self.limit: + v = self.victim() + if v is not None: + log(f"closing a keep-alive connection idle {v.idle_for():.1f}s to make room") + self.active.discard(v) + v.close() + break + waited = time.monotonic() - start + if waited - logged >= 5: + logged = waited + log(f"waiting {waited:.0f}s for a slot; {self.describe()}") + try: + await asyncio.wait_for(self.cond.wait(), 0.25) + except asyncio.TimeoutError: + pass + self.active.add(conn) + if time.monotonic() - start >= 1: + log(f"got a slot after {time.monotonic() - start:.1f}s; {self.describe()}") + + async def release(self, conn): + async with self.cond: + if conn in self.active: + self.active.discard(conn) + self.cond.notify() + + +async def relay(reader, writer, conn, answered): + """Copy until EOF, then half-close the other side.""" + try: + while True: + data = await reader.read(65536) + if not data: + break + conn.touch(answered) + writer.write(data) + await writer.drain() + if writer.can_write_eof(): + writer.write_eof() + except (ConnectionError, OSError): + pass + + +class Ambiguous(Exception): + """The request reached the forward, but no answer came back: it may or + may not have been processed, so it is not sent again.""" + + +async def open_answering(upstream, sent, slots, retry_sent=True): + """Connect to the forward and send the client's first bytes. + + Retries when nothing was sent (the connection failed), and, for a + request that is safe to send again (replayable), when the forward + closes or resets the connection without answering: that is how it + refuses a connection past its limit. Any other request may already + have been processed by then, so it is not sent again: Ambiguous.""" + for attempt in range(RETRIES + 1): + try: + ureader, uwriter = await asyncio.open_connection("127.0.0.1", upstream) + except OSError: + await asyncio.sleep(min(1.0, 0.2 * (attempt + 1))) + continue + if not retry_sent: + try: + uwriter.write(sent) + await uwriter.drain() + except (ConnectionError, OSError): + uwriter.close() + raise Ambiguous() + # Once the connection exists, relay exactly once. Even a GET that + # loses its response must not be replayed in connect-only mode. + return ureader, uwriter, b"" + try: + uwriter.write(sent) + await uwriter.drain() + # A refused forward connection closes (or resets) at once. A + # request still being sent (a large body) gets no answer yet: + # relay it. + first = await asyncio.wait_for(ureader.read(65536), ANSWER_WAIT) + except asyncio.TimeoutError: + if not hedgeable(sent): + log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); relaying without retries") + return ureader, uwriter, b"" + return await hedge(upstream, sent, ureader, uwriter, slots) + except (ConnectionError, OSError): + first = b"" + if first: + return ureader, uwriter, first + uwriter.close() + if not replayable(sent): + raise Ambiguous() + await asyncio.sleep(min(1.0, 0.2 * (attempt + 1))) + return None, None, b"" + + +def replayable(sent): + """A complete GET/HEAD/OPTIONS request without a body (a WebSocket + upgrade included: no answer means no upgrade): sending it again cannot + repeat a change.""" + head = sent.split(b"\r\n\r\n", 1) + return len(head) == 2 and not head[1] and sent.split(b" ", 1)[0] in (b"GET", b"HEAD", b"OPTIONS") + + +def hedgeable(sent): + """A replayable request that is not a WebSocket upgrade: safe to send + again on another connection while the first is still open.""" + return replayable(sent) and b"upgrade: websocket" not in sent.lower() + + +async def hedge(upstream, sent, ureader, uwriter, slots): + """Found live: a request through the forward sometimes got no answer at + all (no error, nothing in the forward's log) and the browser saw a 504. + For a request that is safe to repeat, send it again on new connections + and keep whichever answers first. Each extra connection needs a free + slot (Slots.try_extra); without one, the request just keeps waiting on + the connections it has.""" + pending = {asyncio.ensure_future(ureader.read(65536)): (ureader, uwriter)} + extras = 0 + deadline = time.monotonic() + HEDGE_TOTAL + try: + for extra in range(HEDGES + 1): + if extra < HEDGES: + if slots.try_extra(): + extras += 1 + log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); " + f"sending it again ({extra + 1})") + try: + r, w = await asyncio.open_connection("127.0.0.1", upstream) + w.write(sent) + await w.drain() + pending[asyncio.ensure_future(r.read(65536))] = (r, w) + except (ConnectionError, OSError): + pass + else: + log(f"no answer within {ANSWER_WAIT:.0f}s ({sent[:60]!r}); " + "no free slot to send it again") + wait = ANSWER_WAIT * 2 if extra < HEDGES else max(0.0, deadline - time.monotonic()) + done, _ = await asyncio.wait(pending, timeout=wait, return_when=asyncio.FIRST_COMPLETED) + for task in done: + r, w = pending.pop(task) + try: + data = task.result() + except (ConnectionError, OSError): + data = b"" + if data: + for other, (_, ow) in pending.items(): + other.cancel() + ow.close() + pending.clear() + return r, w, data + w.close() + if not pending: + break + for other, (_, ow) in pending.items(): + other.cancel() + ow.close() + log(f"no answer on {extras + 1} connections ({sent[:60]!r})") + return None, None, b"" + finally: + # The connection that answered (if any) runs on the client's own + # slot; every extra one taken here is closed by now. + await slots.release_extra(extras) + + +async def relay_request(reader, proof_file, subject): + # Only the authenticated loopback proxy may supply the owner subject. + # Replace every caller-supplied proof header before sending to OpenShell. + raw = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 10) + if len(raw) > 65536: + raise ValueError("oversized browser request") + request, *headers = raw[:-4].split(b"\r\n") + checked = [] + owners = [] + websocket = False + for line in headers: + name, sep, value = line.partition(b":") + if not sep or not re.fullmatch(rb"[!#$%&'*+.^_`|~0-9a-zA-Z-]+", name): + raise ValueError("invalid browser header") + if name.lower() == b"x-forwarded-email": + owners.append(value.strip()) + if name.lower() == b"upgrade" and value.strip().lower() == b"websocket": + websocket = True + if name.lower() not in (b"x-saw-relay-identity", b"connection"): + checked.append(line) + if owners != [subject.encode("ascii")]: + raise ValueError("missing authenticated owner") + proof = Path(proof_file).read_bytes() + if not re.fullmatch(rb"[a-f0-9]{64}", proof): + raise ValueError("invalid relay proof") + checked.append(b"Connection: Upgrade" if websocket else b"Connection: close") + checked.append(b"X-Saw-Relay-Identity: " + subject.encode("ascii") + b"." + proof) + return b"\r\n".join([request, *checked]) + b"\r\n\r\n" + + +def handler(upstream, slots, retry_sent=True): + async def handle(creader, cwriter): + conn = None + try: + # The first bytes tell an HTTP request from a WebSocket upgrade, + # and are what a retry resends. + proof_file = os.environ.get("SAW_BROWSER_RELAY_PROOF_FILE") + if proof_file: + sent = await relay_request(creader, proof_file, os.environ["SAW_BROWSER_OWNER_SUBJECT"]) + else: + sent = await creader.read(65536) + if not sent: + return + conn = Conn(b"upgrade: websocket" in sent.lower()) + conn.writers.append(cwriter) + await slots.acquire(conn) + try: + ureader, uwriter, first = await open_answering(upstream, sent, slots, retry_sent) + except Ambiguous: + log("the forward closed without an answer; " + "not sending it again, it may have been processed") + cwriter.write(AMBIGUOUS_ANSWER) + await cwriter.drain() + return + if ureader is None: + log("the forward refused the connection on every retry") + return + conn.writers.append(uwriter) + conn.touch(bool(first)) + if first: + cwriter.write(first) + await cwriter.drain() + await asyncio.gather(relay(creader, uwriter, conn, False), + relay(ureader, cwriter, conn, True)) + uwriter.close() + except (ConnectionError, OSError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError, TimeoutError): + pass + finally: + if conn is not None: + await slots.release(conn) + cwriter.close() + return handle + + +async def main(listen, upstream, limit, retry_sent=True): + server = await asyncio.start_server(handler(upstream, Slots(limit), retry_sent), "127.0.0.1", listen, + limit=BUFFER_LIMIT) + async with server: + await server.serve_forever() + + +if __name__ == "__main__": + listen_port, upstream_port = int(sys.argv[1]), int(sys.argv[2]) + max_conns = int(sys.argv[3]) if len(sys.argv) > 3 else 10 + if len(sys.argv) > 5 or (len(sys.argv) == 5 and sys.argv[4] != "--connect-only"): + raise SystemExit("expected LISTEN_PORT UPSTREAM_PORT [MAX] [--connect-only]") + asyncio.run(main(listen_port, upstream_port, max_conns, retry_sent=len(sys.argv) < 5)) diff --git a/charts/saw-workspace-relay/files/token.py b/charts/saw-workspace-relay/files/token.py new file mode 100644 index 000000000..5cef5e2ae --- /dev/null +++ b/charts/saw-workspace-relay/files/token.py @@ -0,0 +1,63 @@ +"""Refresh the dedicated service-account token in the stock CLI's in-memory config.""" +import base64 +import json +import os +from pathlib import Path +import sys +import time +from urllib.parse import urlencode +from urllib.request import Request, build_opener, HTTPRedirectHandler + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args): + return None + + +def write(path, data): + temp = path.with_suffix(".tmp") + temp.write_text(json.dumps(data)) + temp.replace(path) + + +def refresh(): + cfg = json.loads(Path("/config/relay.json").read_text()) + issuer = cfg["issuer"] + secret = Path("/credential/client-secret").read_text() + request = Request(issuer + "/protocol/openid-connect/token", data=urlencode({ + "grant_type":"client_credentials", "client_id":cfg["clientID"], "client_secret":secret}).encode()) + with build_opener(NoRedirect()).open(request, timeout=12) as response: + result = json.load(response) + token = result["access_token"] + part = token.split(".")[1] + c = json.loads(base64.urlsafe_b64decode(part + "=" * (-len(part) % 4))) + audiences = c["aud"] if isinstance(c["aud"],list) else [c["aud"]] + if (c["sub"] != cfg["subject"] or c["iss"] != issuer or c["azp"] != cfg["clientID"] + or set(audiences) != {cfg["audience"]} + or set(c.get("realm_access",{}).get("roles",[])) != {"openshell-user"} + or c.get("resource_access") or int(c["exp"]) - time.time() < 30): + raise ValueError("unexpected relay claims") + directory = Path("/session/openshell/gateways/relay") + (directory/"mtls").mkdir(parents=True,exist_ok=True) + (directory/"mtls/ca.crt").write_bytes(Path("/config/ca.crt").read_bytes()) + write(directory/"metadata.json", {"name":"relay","gateway_endpoint":cfg["gatewayEndpoint"],"is_remote":False, + "gateway_port":0,"auth_mode":"oidc","oidc_issuer":issuer,"oidc_client_id":cfg["clientID"], + "oidc_audience":cfg["audience"],"oidc_scopes":"openid"}) + write(directory/"oidc_token.json", {"access_token":token,"expires_at":int(c["exp"]), + "issuer":issuer,"client_id":cfg["clientID"]}) + return max(10, int((int(c["exp"])-time.time())/2)) + + +if __name__ == "__main__": + os.umask(0o077) + while True: + try: + delay = refresh() + except Exception as e: + print("Relay token exchange failed: " + type(e).__name__, file=sys.stderr, flush=True) + if sys.argv[1] == "once": + sys.exit(1) + delay = 10 + if sys.argv[1] == "once": + break + time.sleep(delay) diff --git a/charts/saw-workspace-relay/templates/access.yaml b/charts/saw-workspace-relay/templates/access.yaml new file mode 100644 index 000000000..2bf81382f --- /dev/null +++ b/charts/saw-workspace-relay/templates/access.yaml @@ -0,0 +1,132 @@ +{{- if and .Values.enabled .Values.subject }} +{{- $v := .Values }}{{- $n := .Release.Name }}{{- $ui := .Release.Namespace }} +{{- $revision := dict "values" $v "script" (.Files.Get "files/bind.py") | toJson | sha256sum | trunc 10 }} +{{- $script := printf "%s-bind-%s" $n $revision }} +{{- if or (eq $v.subject $v.ownerSubject) (and (not $v.workspaceID) (not $v.publishSubject)) (not $v.ownerSubject) (not $v.gatewayNamespace) }}{{ fail "distinct relay subject, owner and explicit gateway/workspace identities required" }}{{ end }} +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $n }}, namespace: {{ $v.trustedNamespace }}} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $n }}-admin-session, namespace: {{ $v.gatewayNamespace }}} +rules: [{apiGroups: [""], resources: [secrets], resourceNames: [{{ $v.gateway }}-client-ca], verbs: [get, update]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $n }}-admin-session, namespace: {{ $v.gatewayNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $n }}-admin-session} +subjects: [{kind: ServiceAccount, name: {{ $n }}, namespace: {{ $v.trustedNamespace }}}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $n }}-session-holders, namespace: {{ $v.trustedNamespace }}} +rules: [{apiGroups: [""], resources: [pods], verbs: [get]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $n }}-session-holders, namespace: {{ $v.trustedNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $n }}-session-holders} +subjects: [{kind: ServiceAccount, name: {{ $n }}, namespace: {{ $v.trustedNamespace }}}] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $n }}} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $n }}-pull, namespace: {{ $v.imageBuildNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: system:image-puller} +subjects: +- {kind: ServiceAccount, name: {{ $n }}, namespace: {{ $ui }}} +- {kind: ServiceAccount, name: {{ $n }}, namespace: {{ $v.trustedNamespace }}} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $n }}-ingress, namespace: {{ $v.gatewayNamespace }}} +spec: + podSelector: {matchLabels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: gateway}} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $ui }}}} + podSelector: {matchLabels: {app: {{ $n }}, role: relay}} + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $v.trustedNamespace }}}} + podSelector: {matchLabels: {app: {{ $n }}, role: bind}} + ports: [{protocol: TCP, port: 17670}] +{{- range $ns := list $ui $v.trustedNamespace }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $n }}-egress, namespace: {{ $ns }}} +spec: + podSelector: {matchLabels: {app: {{ $n }}, role: {{ ternary "relay" "bind" (eq $ns $ui) }} }} + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: {{ $v.gatewayNamespace }}}} + podSelector: {matchLabels: {app.kubernetes.io/instance: {{ $v.gateway }}, openshell.pattern/role: gateway}} + ports: [{protocol: TCP, port: 17670}] +{{ toYaml (required "issuer and DNS egress required" $v.egress) | indent 2 }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $script }}, namespace: {{ $v.trustedNamespace }}} +immutable: true +data: + bind.py: | +{{ .Files.Get "files/bind.py" | indent 4 }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $script }} + namespace: {{ $v.trustedNamespace }} + annotations: {argocd.argoproj.io/sync-wave: '2'} +spec: + backoffLimit: 2 + activeDeadlineSeconds: 510 + template: + metadata: {labels: {app: {{ $n }}, role: bind}} + spec: + serviceAccountName: {{ $n }} + automountServiceAccountToken: false + restartPolicy: Never + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: bind + image: {{ $v.sdkImage }} + command: [python3, /scripts/bind.py] + env: + - {name: WORKSPACE, value: {{ $v.workspace }}} + - {name: WORKSPACE_ID, value: {{ $v.workspaceID | default "" | quote }}} + - {name: DISCOVER_ID, value: {{ ternary "true" "false" ($v.publishSubject | default false) | quote }}} + - {name: OWNER, value: {{ $v.ownerSubject }}} + - {name: SUBJECT, value: {{ $v.subject }}} + - {name: INSTALLATION, value: {{ $v.installationID }}} + - {name: ENDPOINT, value: {{ $v.gatewayEndpoint }}} + - {name: GATEWAY, value: {{ $v.gateway }}} + - {name: GATEWAY_NAMESPACE, value: {{ $v.gatewayNamespace }}} + - {name: TRUSTED_NAMESPACE, value: {{ $v.trustedNamespace }}} + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_UID, valueFrom: {fieldRef: {fieldPath: metadata.uid}}} + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + volumeMounts: + - {name: script, mountPath: /scripts, readOnly: true} + - {name: identity, mountPath: /identity, readOnly: true} + - {name: admin-session, mountPath: /run/saw-admin} + - {name: api, mountPath: /var/run/secrets/kubernetes.io/serviceaccount, readOnly: true} + volumes: + - {name: script, configMap: {name: {{ $script }}}} + - {name: identity, configMap: {name: {{ $v.gateway }}-gateway-ca, defaultMode: 0440}} + - {name: admin-session, emptyDir: {medium: Memory, sizeLimit: 4Mi}} + - name: api + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: {path: token, expirationSeconds: 600} + - configMap: {name: kube-root-ca.crt, items: [{key: ca.crt, path: ca.crt}]} +{{- end }} diff --git a/charts/saw-workspace-relay/templates/identity.yaml b/charts/saw-workspace-relay/templates/identity.yaml new file mode 100644 index 000000000..36eb66abd --- /dev/null +++ b/charts/saw-workspace-relay/templates/identity.yaml @@ -0,0 +1,151 @@ +{{- if .Values.enabled }} +{{- $v := .Values }}{{- $n := .Release.Name }}{{- $ui := .Release.Namespace }} +{{- range $image := list $v.pythonImage $v.cliImage $v.proxyImage }} +{{- if not (regexMatch "^.+@sha256:[0-9a-f]{64}$" $image) }}{{ fail "relay images must be digest pinned" }}{{ end }} +{{- end }} +{{- $revision := dict "values" $v "script" (.Files.Get "files/identity.py") | toJson | sha256sum | trunc 10 }} +{{- $script := printf "%s-identity-%s" $n $revision }} +{{- if or (eq $ui $v.keycloakNamespace) (eq $ui $v.trustedNamespace) (not (hasPrefix "https://" $v.issuer)) (not $v.clientID) (not $v.workspace) (not $v.installationID) }}{{ fail "dedicated UI namespace, HTTPS issuer and explicit client/workspace scope required" }}{{ end }} +apiVersion: v1 +kind: Secret +metadata: + name: relay-credentials + labels: {saw.redhat.com/runtime-owner: {{ $v.installationID | quote }}} + annotations: {argocd.argoproj.io/sync-options: 'Prune=false,Delete=false'} +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $n }}-browser-proof + namespace: {{ $v.gatewayNamespace }} + labels: {saw.redhat.com/runtime-owner: {{ $v.installationID | quote }}} +type: Opaque +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $n }}-proof, namespace: {{ $v.gatewayNamespace }}} +rules: [{apiGroups: [""], resources: [secrets], resourceNames: [{{ $n }}-browser-proof], verbs: [get, patch]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $n }}-proof, namespace: {{ $v.gatewayNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $n }}-proof} +subjects: [{kind: ServiceAccount, name: {{ $n }}-identity, namespace: {{ $v.keycloakNamespace }}}] +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $n }}-registration-state + namespace: {{ $v.keycloakNamespace }} + labels: {saw.redhat.com/runtime-owner: {{ $v.installationID | quote }}} +type: Opaque +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $n }}-registration-state, namespace: {{ $v.keycloakNamespace }}} +rules: [{apiGroups: [""], resources: [secrets], resourceNames: [{{ $n }}-registration-state], verbs: [get, patch]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $n }}-registration-state, namespace: {{ $v.keycloakNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $n }}-registration-state} +subjects: [{kind: ServiceAccount, name: {{ $n }}-identity, namespace: {{ $v.keycloakNamespace }}}] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: {{ $n }}-identity, namespace: {{ $v.keycloakNamespace }}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $n }}-identity} +rules: +- {apiGroups: [""], resources: [secrets], resourceNames: [relay-credentials], verbs: [get, patch]} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $n }}-identity} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $n }}-identity} +subjects: [{kind: ServiceAccount, name: {{ $n }}-identity, namespace: {{ $v.keycloakNamespace }}}] +--- +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $script }}, namespace: {{ $v.keycloakNamespace }}} +immutable: true +data: + identity.py: | +{{ .Files.Get "files/identity.py" | indent 4 }} +{{- if $v.publishSubject }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: {{ $n }}-public-subject, namespace: {{ $v.trustedNamespace }}} +rules: [{apiGroups: [""], resources: [configmaps], resourceNames: [{{ $v.publicSubjectMap | default "saw-portal-relay-subject" }}], verbs: [get, patch]}] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: {{ $n }}-public-subject, namespace: {{ $v.trustedNamespace }}} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $n }}-public-subject} +subjects: [{kind: ServiceAccount, name: {{ $n }}-identity, namespace: {{ $v.keycloakNamespace }}}] +{{- end }} +{{- $actions := list "create" }} +{{- if $v.cleanupRegistration }} +{{- if or (not (regexMatch "^[0-9a-f]{16}$" $v.cleanupRegistration)) (ne $v.clientID (printf "saw-portal-%s" $v.cleanupRegistration)) }}{{ fail "cleanup requires a registration-specific client ID" }}{{ end }} +{{- $actions = append $actions "delete" }} +{{- end }} +{{- range $action := $actions }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $n }}-{{ $action }}-identity-{{ $revision }} + namespace: {{ $v.keycloakNamespace }} + {{- if eq $action "delete" }} + annotations: {helm.sh/hook: pre-delete, helm.sh/hook-delete-policy: before-hook-creation, helm.sh/hook-weight: '0'} + {{- else }} + annotations: {argocd.argoproj.io/sync-wave: '1'} + {{- end }} +spec: + backoffLimit: 2 + activeDeadlineSeconds: 300 + template: + spec: + restartPolicy: Never + serviceAccountName: {{ $n }}-identity + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + containers: + - name: provision + image: {{ $v.pythonImage }} + command: [python3, /scripts/identity.py] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + env: + {{- if eq $action "delete" }} + - {name: DELETE_BOUND_CLIENT, value: "true"} + - {name: REGISTRATION, value: {{ $v.cleanupRegistration | quote }}} + {{- end }} + {{- if and $v.publishSubject (eq $action "create") }} + - {name: PUBLIC_SUBJECT_MAP, value: {{ $v.publicSubjectMap | default "saw-portal-relay-subject" }}} + - {name: PUBLIC_SUBJECT_NAMESPACE, value: {{ $v.trustedNamespace }}} + {{- end }} + - {name: RUNTIME_NAMESPACE, value: {{ $v.gatewayNamespace | quote }}} + - {name: PROOF_NAME, value: {{ $n }}-browser-proof} + - {name: PROVISIONER_NAMESPACE, value: {{ $v.keycloakNamespace | quote }}} + - {name: REGISTRATION_STATE, value: {{ $n }}-registration-state} + - {name: UI_NAMESPACE, value: {{ $ui }}} + - {name: INSTALLATION, value: {{ $v.installationID | quote }}} + - {name: WORKSPACE, value: {{ $v.workspace | quote }}} + - {name: ISSUER, value: {{ $v.issuer | quote }}} + - {name: CLIENT_ID, value: {{ $v.clientID | quote }}} + - {name: AUDIENCE, value: {{ $v.audience | quote }}} + volumeMounts: + - {name: script, mountPath: /scripts, readOnly: true} + {{- if eq $action "create" }} + - {name: kc, mountPath: /kc-provisioner, readOnly: true} + {{- end }} + volumes: + - {name: script, configMap: {name: {{ $script }}}} + {{- if eq $action "create" }} + - {name: kc, secret: {secretName: {{ required "realm-scoped Keycloak provisioner Secret required" $v.keycloakProvisionerSecret }}, defaultMode: 0440, items: [{key: issuer, path: issuer}, {key: client-id, path: client-id}, {key: client-secret, path: client-secret}]}} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/saw-workspace-relay/templates/ui.yaml b/charts/saw-workspace-relay/templates/ui.yaml new file mode 100644 index 000000000..de1718cd5 --- /dev/null +++ b/charts/saw-workspace-relay/templates/ui.yaml @@ -0,0 +1,144 @@ +{{- if and .Values.enabled .Values.subject }} +{{- $v := .Values }}{{- $n := .Release.Name }} +apiVersion: v1 +kind: ConfigMap +metadata: {name: {{ $n }}} +data: + relay.json: {{ pick $v "issuer" "clientID" "subject" "audience" "gatewayEndpoint" | toJson | quote }} + users: {{ printf "%s\n" $v.ownerSubject | quote }} + ca.crt: | +{{ $v.gatewayCA | indent 4 }} + token.py: | +{{ .Files.Get "files/token.py" | indent 4 }} + saw_ui_limit.py: | +{{ .Files.Get "files/saw_ui_limit.py" | indent 4 }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: {{ $n }}-router} +spec: + podSelector: {matchLabels: {app: {{ $n }}, role: relay}} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: {matchLabels: {policy-group.network.openshift.io/ingress: ""}} + ports: [{protocol: TCP, port: 4180}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $n }} + annotations: {argocd.argoproj.io/sync-wave: '3'} +spec: + replicas: {{ ternary 0 1 (.Values.paused | default false) }} + selector: {matchLabels: {app: {{ $n }}, role: relay}} + template: + metadata: + labels: {app: {{ $n }}, role: relay} + annotations: {checksum/config: {{ printf "%s%s%s" ($v | toJson) (.Files.Get "files/token.py") (.Files.Get "files/saw_ui_limit.py") | sha256sum }}} + spec: + serviceAccountName: {{ $n }} + automountServiceAccountToken: false + securityContext: {runAsNonRoot: true, seccompProfile: {type: RuntimeDefault}} + initContainers: + - name: initial-token + image: {{ $v.pythonImage }} + command: [python3, /config/token.py, once] + securityContext: &restricted {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + volumeMounts: &tokenMounts + - {name: config, mountPath: /config, readOnly: true} + - {name: credential, mountPath: /credential, readOnly: true} + - {name: session, mountPath: /session} + containers: + - name: token-refresh + image: {{ $v.pythonImage }} + command: [python3, /config/token.py, refresh] + securityContext: *restricted + volumeMounts: *tokenMounts + readinessProbe: + exec: + command: [python3, -c, "import urllib.request; assert urllib.request.urlopen('http://127.0.0.1:28789/',timeout=4).status==200"] + timeoutSeconds: 6 + periodSeconds: 15 + - name: forward + image: {{ $v.cliImage }} + command: [/bin/sh, -c] + args: + - | + while true; do + openshell forward service -g relay --workspace "$1" "$2" --target-port 18789 --local 127.0.0.1:28789 + sleep 2 + done + - relay + - {{ $v.workspace }} + - {{ $v.sandbox }} + env: [{name: XDG_CONFIG_HOME, value: /session}, {name: HOME, value: /session}] + securityContext: *restricted + volumeMounts: [{name: session, mountPath: /session}] + - name: transport + image: {{ $v.pythonImage }} + # token.py shares this ConfigMap; isolate stdlib imports from /config. + command: [python3, -I, /config/saw_ui_limit.py, '18789', '28789', '16', --connect-only] + securityContext: *restricted + env: + - {name: SAW_BROWSER_OWNER_SUBJECT, value: {{ $v.ownerSubject | quote }}} + - {name: SAW_BROWSER_RELAY_PROOF_FILE, value: /proof/relay-proof} + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: proof, mountPath: /proof, readOnly: true} + - name: browser-login + image: {{ $v.proxyImage }} + args: + - --http-address=0.0.0.0:4180 + - --upstream=http://127.0.0.1:18789 + - --provider=oidc + - --oidc-issuer-url={{ $v.issuer }} + - --client-id=openshell-dashboard + - --client-secret-file=/dev/null + - --code-challenge-method=S256 + - --redirect-url=https://{{ $v.host }}/oauth2/callback + - --scope=openid email profile + - --oidc-email-claim=sub + - --authenticated-emails-file=/config/users + - --insecure-oidc-allow-unverified-email=true + - --pass-user-headers=true + - --skip-provider-button=true + - --request-logging=false + - --reverse-proxy=true + - --cookie-secure=true + - --cookie-refresh=60s + - --cookie-name=_saw_relay_session + env: + - name: OAUTH2_PROXY_COOKIE_SECRET + valueFrom: {secretKeyRef: {name: relay-credentials, key: cookie-secret}} + ports: [{name: http, containerPort: 4180}] + readinessProbe: {httpGet: {path: /ping, port: http}} + securityContext: *restricted + volumeMounts: [{name: config, mountPath: /config, readOnly: true}] + volumes: + - {name: session, emptyDir: {medium: Memory, sizeLimit: 2Mi}} + - {name: config, configMap: {name: {{ $n }}}} + - name: proof + secret: {secretName: relay-credentials, items: [{key: relay-proof, path: relay-proof}]} + - name: credential + secret: {secretName: relay-credentials, defaultMode: 0440, items: [{key: client-secret, path: client-secret}]} +--- +apiVersion: v1 +kind: Service +metadata: {name: {{ $n }}} +spec: + selector: {app: {{ $n }}, role: relay} + ports: [{port: 4180, targetPort: http, name: http}] +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ $n }} + labels: {saw.redhat.com/oidc-redirect: "true"} + annotations: {argocd.argoproj.io/sync-wave: '4', haproxy.router.openshift.io/set-forwarded-headers: replace} +spec: + host: {{ required "UI hostname required" $v.host }} + to: {kind: Service, name: {{ $n }}} + port: {targetPort: http} + tls: {termination: edge, insecureEdgeTerminationPolicy: Redirect} +{{- end }} diff --git a/charts/saw-workspace-relay/values.yaml b/charts/saw-workspace-relay/values.yaml new file mode 100644 index 000000000..f597cf7a8 --- /dev/null +++ b/charts/saw-workspace-relay/values.yaml @@ -0,0 +1,26 @@ +enabled: false +clientID: "" +keycloakNamespace: "" +keycloakProvisionerSecret: "" +issuer: "" +audience: openshell-cli +gatewayEndpoint: "" +gatewayCA: "" +workspace: "" +workspaceID: "" +sandbox: "" +ownerSubject: "" +ownerLogin: "" +installationID: "" +trustedNamespace: "" +subject: "" +host: "" +# Explicit DNS, OIDC issuer and Kubernetes API Service/backend destinations and +# ports. The membership Job needs API access for temporary public trust and Pod +# identity checks; reuse the controller's approved bootstrap API egress rules. +# These rules also apply to the browser relay, which has no Kubernetes API token. +egress: [] +sdkImage: "" +pythonImage: registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c +cliImage: quay.io/opendatahub/odh-openshell-cli@sha256:2b6d420543ce93825092704a52127318fa90fd00df4a07d739efcd6c9f20ea57 +proxyImage: quay.io/oauth2-proxy/oauth2-proxy@sha256:37c1570c0427e02fc7c947ef2c04e8995b8347b7abc9fcf1dbb4e376a4b221a7 diff --git a/cli/src/openshell_saw/cli.py b/cli/src/openshell_saw/cli.py index 5a1025a97..675f1bf3c 100644 --- a/cli/src/openshell_saw/cli.py +++ b/cli/src/openshell_saw/cli.py @@ -8,6 +8,7 @@ import click from . import config, helm, kube, oidc +from .workspace import workspace @click.group() @@ -30,6 +31,9 @@ def main(ctx, namespace, shared_namespace, ssh_key): ctx.obj["cfg"] = cfg +main.add_command(workspace) + + # --- OIDC commands (for provisioning — users authenticate via 'openshell login') --- diff --git a/cli/src/openshell_saw/workspace.py b/cli/src/openshell_saw/workspace.py new file mode 100644 index 000000000..4419271fa --- /dev/null +++ b/cli/src/openshell_saw/workspace.py @@ -0,0 +1,77 @@ +"""Owner commands use the same RHDH templates and status service as the browser.""" +import json +import re +from urllib.parse import urlsplit + +import click +import requests + + +@click.group() +@click.option("--url", envvar="SAW_RHDH_URL", required=True, help="HTTPS origin of Developer Hub.") +@click.option("--token-file", envvar="SAW_RHDH_TOKEN_FILE", required=True, type=click.File("r"), + help="File containing a Backstage identity token. Use - for stdin.") +@click.pass_context +def workspace(ctx, url, token_file): + """Manage your workspace through Developer Hub.""" + parsed = urlsplit(url) + if (parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password + or parsed.path not in ("", "/") or parsed.query or parsed.fragment): + raise click.UsageError("Provide the HTTPS origin of Developer Hub.") + token = token_file.read().strip() + if not token or any(char.isspace() for char in token): + raise click.UsageError("Expected one identity token.") + ctx.obj = (url.rstrip("/"), token) + + +def call(context, path, payload=None): + url, token = context + headers = {"Authorization": "Bearer " + token} + if path.startswith("/api/proxy/saw-status/"): + headers["X-Saw-Token"] = token + try: + response = requests.request("GET" if payload is None else "POST", url + path, + headers=headers, json=payload, timeout=30, allow_redirects=False) + except requests.RequestException as exc: + raise click.ClickException("Developer Hub request failed") from exc + if not 200 <= response.status_code < 300: + raise click.ClickException(f"Developer Hub returned HTTP {response.status_code}") + return response.json() + + +@workspace.command() +@click.pass_obj +def status(context): + """Show desired and observed workspace state.""" + click.echo(json.dumps(call(context, "/api/proxy/saw-status/workspace"), indent=2)) + + +@workspace.command() +@click.argument("run") +@click.pass_obj +def logs(context, run): + """Read the provisioning log for one of your Tekton runs.""" + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,61}[a-z0-9]", run): + raise click.UsageError("Expected a Tekton run name.") + click.echo(call(context, "/api/proxy/saw-status/run/" + run)["text"]) + + +def set_state(context, operation): + task = call(context, "/api/scaffolder/v2/tasks", { + "templateRef": "template:default/set-saw-workspace-state", + "values": {"operation": operation}, "secrets": {}}) + click.echo(json.dumps({"task": task["id"]})) + + +@workspace.command() +@click.pass_obj +def stop(context): + """Stop the sandboxes and retain their storage.""" + set_state(context, "stop") + + +@workspace.command() +@click.pass_obj +def start(context): + """Resume the registered sandboxes.""" + set_state(context, "start") diff --git a/docs/container-images.md b/docs/container-images.md index 06e286c8f..ca73cae31 100644 --- a/docs/container-images.md +++ b/docs/container-images.md @@ -20,9 +20,10 @@ those deployment overrides local when preparing changes for upstream. | `nemoclaw-sandbox` | `latest`, `sha-` | Data-science default and CUDA profiles | | `governance-interceptor` | `latest`, `sha-`, OpenShell ref (currently `v0.1.6`) | Interceptor deployment and image builder | | `saw-credential-proxy` | `latest`, `sha-` (multi-arch) | Chart default `credentialProxy.image` (digest pinned in `charts/openshell-saw/values.yaml`) | +| `saw-kubernetes-runtime` | `latest`, `sha-` (multi-arch) | Container reconciler, preparation/enrollment helpers, standalone projector and relay membership Job; operator site values pin the manifest digest. | | `saw-harness-` | Bundle version, `sha-` | Optional digest-pinned `harnessRef.image` | -`openshell-gateway`, `nemoclaw-cli`, `nemoclaw-sandbox`, `saw-credential-proxy` and `governance-interceptor` tags are +`openshell-gateway`, `nemoclaw-cli`, `nemoclaw-sandbox`, `saw-credential-proxy`, `saw-kubernetes-runtime` and `governance-interceptor` tags are multi-architecture indexes (`linux/amd64`, `linux/arm64`): each architecture builds natively and pushes by digest, and the merge job joins the digests with `.github/actions/merge-image`. `make images-mirror` on an arm64 cluster gets @@ -50,6 +51,14 @@ repository. GHCR can create packages as private initially: a package administrat must set each package's visibility to **Public** in its GitHub package settings, including newly added harness bundles. Public pulls need no registry Secret. Publishing uses `GITHUB_TOKEN` with `packages: write`. +The Kubernetes runtime recipe is `kubernetes-runtime/Containerfile`; it packages +the canonical BOM and temporary admin-session modules, the workspace projector, +and protobuf stubs generated from the pinned OpenShell source. Its build context +allowlist excludes unrelated repository and local files. The +[operator build procedure](#building-a-container-backend-candidate) below +publishes a candidate to an OpenShift internal registry; the +[installation guide](container-installation.md#pin-the-built-components) +maps its digests into the site inputs. The first publication of a new package can therefore fail its public-pull check after the image is pushed. Set it to Public, then rerun the workflow. GitHub's documented [package REST API](https://docs.github.com/en/rest/packages/packages) @@ -112,8 +121,13 @@ skopeo inspect --no-creds docker://ghcr.io/validatedpatterns-sandbox/nemoclaw-cl Registry tooling verifies TLS. Route-based mirroring and publishing discover the ingress CA through the authenticated `oc` session. In-cluster mirror Jobs and the -golden-image import use the OpenShift service CA for the internal registry. Local -registry commands also accept `REGISTRY_CA_BUNDLE=/path/to/ca.pem` for private +golden-image import use the OpenShift service CA for the internal registry. + +Local Linux mirroring uses `oc registry login`, which needs a bearer token. On a +bastion, check `oc whoami --show-token >/dev/null` first. A certificate-based +kubeconfig can access the Kubernetes API but cannot supply that token. Use the +documented `oc login` step for the target cluster before `make images-mirror`. +Local registry commands also accept `REGISTRY_CA_BUNDLE=/path/to/ca.pem` for private registries; the bundle supplements system trust. CA lookup or certificate failures never enable an insecure retry. @@ -138,6 +152,129 @@ create a Docker registry Secret named `ghcr-push-secret` in `BUILD_NS`, or overr and needs no GHCR push credentials. A fork interceptor build also needs `charts/governance-interceptor`'s `image` override. +## Building a container backend candidate + +PR CI builds images but does not publish them. For an unmerged candidate, use +the following from a clean checkout of the reviewed commit, already pushed to +`SAW_REPO`. The runtime BuildConfig fetches that exact commit; the governance +chart embeds this checkout's canonical Dockerfile and extension into its build +inputs. Its OpenShell source reference remains the one in the chart values. +No local credential files enter either build context. + +Use an operator-owned build namespace and an explicitly selected cluster +context. These builds push to the internal registry with the build service +account; no external registry credential is required. Each build produces one +architecture. Use a matching cluster architecture, or constrain both BuildConfigs' +`spec.nodeSelector` to the intended `kubernetes.io/arch` before starting them. + +```sh +set -eu +SAW_REPO='https://github.com//secure-agent-workspace.git' +SAW_REV=$(git rev-parse HEAD) +test -z "$(git status --porcelain)" +BUILD_NS=saw-component-builds +BUILD_ID=$(printf '%s' "$SAW_REV" | cut -c1-12) +oc create namespace "$BUILD_NS" --dry-run=client -o yaml | oc apply -f - + +cat > runtime-build.yaml < governance-build.yaml +oc -n "$BUILD_NS" apply -f governance-build.yaml + +RUNTIME_BUILD=$(oc -n "$BUILD_NS" start-build "runtime-${BUILD_ID}" -o name) +GOVERNANCE_BUILD=$(oc -n "$BUILD_NS" start-build "governance-${BUILD_ID}" -o name) +oc -n "$BUILD_NS" wait "$RUNTIME_BUILD" \ + --for=jsonpath='{.status.phase}'=Complete --timeout=30m +oc -n "$BUILD_NS" wait "$GOVERNANCE_BUILD" \ + --for=jsonpath='{.status.phase}'=Complete --timeout=30m + +RUNTIME_DIGEST=$(oc -n "$BUILD_NS" get "$RUNTIME_BUILD" \ + -o jsonpath='{.status.output.to.imageDigest}') +GOVERNANCE_DIGEST=$(oc -n "$BUILD_NS" get "$GOVERNANCE_BUILD" \ + -o jsonpath='{.status.output.to.imageDigest}') +test "$(printf '%s\n' "$RUNTIME_DIGEST" "$GOVERNANCE_DIGEST" \ + | grep -Ec '^sha256:[a-f0-9]{64}$')" = 2 +RUNTIME_IMAGE="image-registry.openshift-image-registry.svc:5000/${BUILD_NS}/saw-kubernetes-runtime@${RUNTIME_DIGEST}" +GOVERNANCE_IMAGE="image-registry.openshift-image-registry.svc:5000/${BUILD_NS}/governance-interceptor@${GOVERNANCE_DIGEST}" +printf '%s\n' "$SAW_REV" "$RUNTIME_IMAGE" "$GOVERNANCE_IMAGE" +``` + +Expected: both Builds are `Complete`, followed by two digest-pinned image +references. A failed build does not qualify a tag left by an earlier build. +Inspect that exact Build with `oc -n "$BUILD_NS" logs "$RUNTIME_BUILD"` +(or `"$GOVERNANCE_BUILD"`); fix its source and build the new reviewed revision. +These commands build artifacts only; they do not update running workspaces. + +Keep the generated build manifests in the operator's configuration repository +when GitOps owns builds. Give each recipe revision its own BuildConfig and source +ConfigMap names, as above. Consumers use the captured digests, never a moving tag. +Record the Build names, source SHA and output digests outside the product PR. +After recording them, remove the two local generated YAML files. Retain the +published image blobs while any site references their digests. + +## Prepare the data-science notebook for container workspaces + +The shipped notebook image sets `WORKDIR /sandbox` with mode `0770` and +ownership `1000:0`. Container workspaces run with the OpenShift namespace UID +and group. They need search permission on this directory to start the sandbox +entrypoint. Prepare the image once in the same build namespace as the runtime: + +```sh +make container-notebook-image BUILD_NS="$BUILD_NS" BUILD_ARCH=amd64 > notebook-site.yaml +cat notebook-site.yaml +``` + +Use `arm64` for ARM workers. The target reads the pinned notebook image from +the BOM, builds on the selected architecture, adds only directory traversal +(`chmod o+x /sandbox`), and restores the source image's declared non-root +`USER`. It prints the digest from the completed Build and the ImageStream +name used for scoped pull access. Each output supplies one architecture. + +Merge the emitted `profileImageOverrides.data-science.notebook` and +`imagePull` fields into each container owner's site before rendering or +approving the BOM. Preserve other named sandbox overrides and ImageStream +entries. Use the same notebook digest in the image-cache values if enabled, +and grant its service account pull access to the emitted ImageStream as +described in the installation guide. A changed notebook digest requires a +new profile approval. VM installations retain their regular image selection. + ## Releasing a credential proxy change The chart default is the published multi-arch index. Its adjacent source comment diff --git a/docs/container-installation.md b/docs/container-installation.md new file mode 100644 index 000000000..ff5ed7b3c --- /dev/null +++ b/docs/container-installation.md @@ -0,0 +1,779 @@ +# Installing container workspaces + +Container mode adds a backend to the existing SAW installation and profile +model. VM remains the default. The [architecture](container-workspaces.md) +describes components, trust boundaries and data paths; this page describes +operator inputs and installation. There is no additional SAW workspace CRD. + +## Choose the installation owner + +| Entry point | Desired configuration | Component lifecycle | +|---|---|---| +| Validated Pattern | Static entries in `overrides/saw-users.yaml`, with `projection.enabled: true` | Pattern owns the root chart; its ApplicationSet and Argo own container component applications. | +| Standalone Make | Operator-owned values file; `DEPLOYMENT_MODE=container` | Helm owns one root `saw-users` release; ApplicationSet and Argo own component applications. Developer Hub is not required. | +| RHDH self-service | Authenticated create/update/delete request plus operator-provided `portal.containerSites` | Existing portal pipelines and generator own the registration; Argo uses the same component charts. | + +Use one entry point per registration. Do not Helm-install a workspace already +owned by the pattern or RHDH. Both VM and container users can coexist in a +`saw-users` installation. Each registration has one backend; it is not an +in-place VM-to-container migration setting. + +## Shared platform prerequisites + +Start with the SAW platform services: Keycloak/owner accounts, governance, +Vault and External Secrets, image availability and OpenShift GitOps. The +container Make path uses GitOps even when the root chart is installed manually. +It does not require KubeVirt for its workload. Keep KubeVirt installed when also +hosting VM workspaces. + +Additional requirements: + +- OpenShift with enforced `AdminNetworkPolicy` and native + `ValidatingAdmissionPolicy`; the baseline is OpenShift 4.22+ with OVN-Kubernetes. +- The pinned stock Agent Sandbox controller and `agents.x-k8s.io/v1beta1` API. + Install [charts/agent-sandbox](../charts/agent-sandbox) once per cluster, or + reuse a compatible platform-owned installation. Do not adopt an existing + release or replace its CRDs from a per-workspace install. +- Argo CD ApplicationSet plugin generators, a project permitting the configured + destinations/resource kinds, and permission for Argo to manage the workload, + control, UI and history namespaces. The standalone projector's namespace is + separate from Argo CD and labelled `argocd.argoproj.io/managed-by`. +- RWO storage for gateway state, workspace data, controller ledger and audit + history; a shared filesystem class for the preparation/audit exchange PVCs. +- Reachable DNS, Kubernetes API, trusted OIDC issuer, governance, registry and + approved provider endpoints. Use the [container topology preflight](#discover-and-review-container-egress) + to derive API, DNS and issuer rules, or supply those rules explicitly. + Allocate a distinct AdminNetworkPolicy priority for each workspace. +- `oc`, Helm 3 and `uv` on the installation machine. Provider credentials stay + in Vault; local files and Git contain references and public configuration. + +Argo CD can retain an older list of API versions after the stock Agent Sandbox +CRD is installed. For a generated gateway Application, the parent supplies the +two API versions used by its Helm chart when the parent, gateway and identity +charts share one immutable revision. Mixed revisions retain normal Helm +discovery. The identity Job reads the live API +server over verified TLS and checks the group, version, resource, kind and scope +of both Sandbox and AdminNetworkPolicy before creating or projecting any keys. +Missing discovery stops identity bootstrap. The policy chart also applies the +actual AdminNetworkPolicy objects. Direct gateway Helm installs keep their +existing capability checks. + +If the cluster uses a private CA, configure client trust with a combined bundle +containing the normal system roots **and** the independently verified cluster CA. +For example, set `SSL_CERT_FILE` to that combined file for Python/`uv` commands. +A cluster-only bundle removes public trust and can break dependency downloads; +do not disable certificate verification to work around this. + +For a platform that does not already own Agent Sandbox, its manual installation +is: + +```sh +helm upgrade --install agent-sandbox charts/agent-sandbox \ + --namespace agent-sandbox-system --create-namespace \ + --set enabled=true --wait +``` + +For a GitOps platform, declare the same chart as a platform Application. +Provision its destination namespace `agent-sandbox-system` in the cluster group +or enable `CreateNamespace=true` on that Application. The Helm command above +creates the namespace with `--create-namespace`. The chart +source lock pins the upstream manifest and controller digest. Removing an +individual workspace must not uninstall this cluster-wide dependency. + +## Public inputs and credential references + +For Kata workload placement, follow the [Kata installation and Developer Hub guide](kata-workspaces.md). Both modes use the same Kubernetes lifecycle and workspace BOM. + +The same site object is used as `containerSites.` for static installs or +`portal.containerSites.` for RHDH. It is administrator-owned, not a form +field. Start from the component charts' values and schemas: + +| Site input | Contents / source | +|---|---| +| `subject` | Resolved Keycloak subject UUID matching the owner. Static inputs supply it explicitly; RHDH resolves the authenticated owner and requires a matching site. | +| `stack.identity` | Pinned revision and [identity chart values](../charts/saw-kubernetes-identity/values.yaml): installation identity, gateway SANs and stock generator/helper images. Administrator sessions use the VM's shared temporary-identity mechanism; no administrator certificate or renewal input is supplied. | +| `stack.policy` | Pinned revision and [admission/network values](../charts/saw-runtime-policies/values.yaml): runtime namespace, service accounts, explicit egress and unique priority. | +| `stack.proxy` | Pinned revision and [credential-proxy values](../charts/saw-kubernetes-proxy/values.yaml): trusted namespace and egress. Provider bindings/digest derive from the shared catalog. | +| `stack.gateway` | Pinned revision and [gateway values](../charts/openshell-saw-kubernetes/values.yaml): stock gateway/driver images, storage, OIDC and network configuration. | +| `stack.audit` | Pinned revision and [audit values](../charts/saw-audit-writer/values.yaml): history/exchange storage classes, image and owner identity. Targets derive from the BOM. | +| `governanceNamespace`, `providerAllowHosts` | Shared governance location and permitted provider destinations. | +| `automaticEnrollment` | `enabled`, pinned helper image/revision, build namespace, resource limits, approved profile digests and reviewed boundary reference. | +| `automaticEnrollment.relay` | Pinned relay chart revision, issuer, Keycloak namespace and realm registrar Secret **name**, and explicit DNS, OIDC and Kubernetes API egress. See [relay values](../charts/saw-workspace-relay/values.yaml). | +| `profileImageOverrides`, `profileBoundaries` | Optional per-profile pinned image overrides and explicit provider/boundary approval. Changing the profile requires its own approval. | +| `imagePull` | Optional namespace and ImageStream names for access to an OpenShift internal registry. | + +RHDH offers **Container** on the create form when `portal.deploymentModes` +includes `container` and at least one configured owner site has a catalog profile +in `automaticEnrollment.approvedProfiles`. The form lists the union of those +profiles. The registration pipeline checks the signed-in owner's own site and +approved profile. For a first registration, pin the reviewed profile digest +and keep `automaticEnrollment.enabled: false` until the operator reviews +the live foundation. Static GitOps and Make can start that review without an +approved profile digest. + +The portal checks the current Keycloak subject before changing a container's +registration or Vault credentials. Configure `portal.adminSubjects` with the +immutable Keycloak subject for each portal administrator who may act for another +owner. The portal Task needs the existing read-only directory client projection +`saw-generator-keycloak` when container sites are enabled. + +### Prepare large images on cold nodes + +The pinned stock OpenShell driver rolls back a sandbox whose bootstrap remains +unfinished for five minutes. A slow first image pull can consume that entire +window. Check cold nodes when qualifying a profile. For large images, install +the optional `charts/saw-image-cache` chart through the clustergroup before +creating workspaces. Give it the same immutable digests selected by the BOM +and site overrides. [Pinned driver](https://github.com/opendatahub-io/openshell/blob/305c8b3d7bbdb97a6f2f7264c3ad43b0087c307f/crates/openshell-driver-kubernetes/src/driver.rs#L284) + +```yaml +# Site values +clusterGroup: + namespaces: + saw-image-cache: + operatorGroup: false + applications: + saw-image-cache: + name: saw-image-cache + namespace: saw-image-cache + path: charts/saw-image-cache + extraValueFiles: + - /overrides/workspace-image-cache.yaml +``` + +```yaml +# overrides/workspace-image-cache.yaml +enabled: true +images: + - /@sha256:<64-hex-digest> +nodeSelector: + kubernetes.io/os: linux +``` + +Kubelet pulls the images as read-only OCI image volumes. The DaemonSet runs a +separate nonroot holder with an immutable image, no service-account token, +dropped capabilities, a read-only root filesystem and no pod network access. +The workload images are never executed by the holder. Set `imagePullSecrets` +to authorized namespaced references for a private registry. For OpenShift +internal images in another namespace, grant the holder service account only +`get` on the required `imagestreams/layers` resource names. +[Image volume behavior](https://v1-35.docs.kubernetes.io/docs/concepts/storage/volumes/#image) + +```sh +oc -n saw-image-cache rollout status daemonset/saw-image-cache --timeout=30m +oc -n saw-image-cache get daemonset/saw-image-cache -o json | + jq -e '.status.desiredNumberScheduled > 0 and + .status.numberReady == .status.desiredNumberScheduled and + .status.updatedNumberScheduled == .status.desiredNumberScheduled' +``` + +Confirm that the ready holders cover every node eligible for the workload. +Match workload selectors and tolerations, and recheck after adding nodes or +changing digests. Each node needs space for the selected images. Keep the +holder running while the profile is in use. + +### One-time Keycloak registration setup + +Use a dedicated Keycloak realm. Run this once from the checkout with an +explicit kubeconfig for the intended cluster. The bootstrap Job needs the +existing master bootstrap Secret only during setup. If the issuer requires +a private CA, supply `--ca-configmap NAME` with key `ca-bundle.crt`. + +```sh +python3 scripts/portal-container/setup-relay-provisioner.py \ + --namespace "$KEYCLOAK_NAMESPACE" --issuer "$KEYCLOAK_ISSUER" \ + --bootstrap-secret "$PLATFORM_BOOTSTRAP_SECRET" | + oc --kubeconfig="$KUBECONFIG" apply --server-side -f - +oc --kubeconfig="$KUBECONFIG" -n "$KEYCLOAK_NAMESPACE" wait \ + --for=condition=Complete job/saw-relay-provisioner --timeout=4m +``` + +Setup creates a confidential service account with only the realm-management +`create-client` permission. It registers an administrator-defined client scope +`saw-workspace-relay` for the OpenShell audience and user role, and allows that +scope for authenticated dynamic registration. Its credential is stored in +`saw-relay-provisioner` in the Keycloak namespace. Set +`automaticEnrollment.relay.keycloakProvisionerSecret: saw-relay-provisioner` +and use a full Git commit for `automaticEnrollment.relay.revision`. + +The registrar can create clients in the workspace realm. The Job enforces the +registration-specific client name. Protect the registrar Secret and restrict who +can schedule Jobs in its namespace. The cleanup Job mounts only its script and +uses its service account to read the individual registration receipt. + +Relay Jobs use the registration API to create one client and retain its +per-client registration access token in the Keycloak namespace for deletion. +An ambiguous creation without a registration receipt requires administrator +recovery of that exact client. A client ID collision fails closed. The browser +Pods receive only their own login/forwarding material. + +The Kubernetes runtime and governance image workflows run pull-request builds +with read-only repository access. Only main-branch publication and manifest +merge jobs receive package-write permission. + +### Admission and upgrade + +Supply digest-pinned images for enabled sandbox slots through the shared BOM +or its site image overrides. The composition derives the allowed slot image, +runtime bootstrap, supervisor command and per-slot claims from those inputs. +The five admitted Pod classes are the gateway StatefulSet, Sandbox workload, +supervisor, gate-preparation Job and audit-forwarder ReplicaSet. Unknown Pods, +foreign Secret projections and in-place executable changes are denied. +The gateway can remove scheduling gates. It can create internal boundary +Services and immutable Pod-owned bootstrap Secrets. Argo installs the stock +network fences and the gateway can only read them. The static projector may +create only its named Opaque plugin credential in the two designated namespaces. + +Re-review and capture the revised admission policies, bindings, gateway Role +and network fences during foundation installation. Existing published +boundaries and compiled plans reference the old specifications. Qualify a new +registration before migrating an existing installation. For registrations made +with the old relay scheme, retire the old client with the old installation's +cleanup procedure before replacing its credentials. + +Component `values` accept the composition placeholders `@OWNER@`, `@SUBJECT@`, +`@INSTALLATION@`, `@RUNTIME@`, `@TRUSTED@`, `@ARCHIVE@`, `@UI@` and `@DOMAIN@`. +The composition substitutes these as data; it does not evaluate user input as +Helm templates. Do not author `targets`, `enrollment`, `providerPolicy` or relay +subjects manually; the common projector and trusted bootstrap derive them. + +`automaticEnrollment.relay.egress` must permit the cluster's Kubernetes API +Service/backends and their ports, using the same approved destinations as the +controller's bootstrap egress, as well as DNS and the OIDC issuer. The relay +membership Job reads Pod identities and updates the named public client-trust +Secret before connecting to OpenShell. DNS/OIDC access alone cannot complete +that Job. The gateway can create the workspace before the reconciler grants the +owner its administrator role. The binding Job waits up to 210 seconds after +authentication for both observations, pins the workspace UUID once observed, +and rejects a changed UUID or unexpected owner role. The browser relay has no Kubernetes token or trust-write RBAC. Changing +the relay inputs creates a new input-hashed binding Job through GitOps. + +For sites with split-horizon DNS, run the [topology preflight](#discover-and-review-container-egress) +from a trusted in-cluster Pod. The gateway and relay need the issuer address +that Pod resolves on TCP 443. A successful browser login from the workstation +does not test this path. + +Public trust publication is asynchronous: the Kubernetes Secret volume must +refresh, then the stock gateway must reload it. Initial TLS failures, including +`BadSignature` while an older temporary CA is still mounted, can occur during +this interval. Keep the configured bounded authenticated readiness wait; do not +disable certificate verification. Confirm a subsequent successful operation and +fresh controller progress. Persistent errors or an exhausted wait are failures, +not evidence of a harmless delay. + +Static registration shape, to merge with the complete operator site values: + +```yaml +global: + repoURL: https://github.com/your-org/secure-agent-workspace.git + targetRevision: + vpArgoNamespace: vp-gitops + clusterDomain: cluster.example.com # ingress is apps.cluster.example.com +projection: + enabled: true + applicationSetControllerLabels: + app.kubernetes.io/name: + egress: # explicit Kubernetes API and DNS NetworkPolicy rules + - to: + - ipBlock: {cidr: /32} + ports: [{protocol: TCP, port: 6443}] + # Add the cluster's API Service and DNS paths as required by its networking. +users: + - name: alice + ownerSubject: + registration: <16-lowercase-hex-characters> + installationID: + vaultPrefix: secret/data/hub/saw-alice/ + profiles: [data-science] + pruneOnRemove: true + intentGeneration: 1 + lifecycleVersion: 1 + desiredState: Running + restartGeneration: 1 + initialModelInputs: # public selectors from the selected BOM's catalog + inference.model: + inference.cuda_model: + modelInputs: # initially equal to initialModelInputs + inference.model: + inference.cuda_model: + values: + deployment: {mode: container} +containerSites: + alice: + subject: + # Complete stack, network, storage, provider and relay inputs from the table. + automaticEnrollment: + enabled: false # foundation review; enable after approval + # Complete pinned helper, resources and relay inputs from the table. +``` + +This shows the input contract, not a runnable site with omitted fields. Generate +unique registration/installation IDs and keep them stable for the installation. +Never reuse an old registration to adopt retained storage or credentials. + +Enrollment also pins the initial compiled plan. A runtime/compiler change that +changes that plan cannot be authorized merely by changing an image digest, +source revision or intent generation. An in-place operator reapproval/migration +workflow is not implemented. Use the documented deletion and a new registration +for such a change; runtime/workspace storage is removed, while the separate +audit history remains retained. Do not remove the plan check or edit the +immutable enrollment publication. Ordinary supported public-model updates use +the existing model-input lifecycle and do not require a new registration. + +For an internal-registry runtime image, `automaticEnrollment.buildNamespace` +identifies its build namespace. The enrollment and preparation pull grants are +restricted to the image stream named by the configured digest-pinned image. + +The selected BOM determines provider Secret names and fields. For example, with +the default NVIDIA-backed `data-science` profile, store its key in +`secret/data/hub/saw-alice//inference` as `api_key`. Configure any +additional enabled provider inputs in the same registration prefix. RHDH uses +the existing request-to-Vault pipeline; standalone operators use their normal +Vault provisioning. Neither the Make helper nor projector loads raw keys. + +Native OpenAI, Anthropic and NVIDIA inference use the same provider profiles as +VMs. Anthropic uses its Messages protocol; NVIDIA model selectors preserve the +vendor/model slash. Select the provider in the approved BOM, then reapprove its +destination and profile inputs through the normal enrollment path. See +[provider qualification](provider-driver-qualification.md) for the BOM shape, +probe commands and current live-test limits. + +## Pin the built components + +Use published release digests or follow the +[candidate build commands](container-images.md#building-a-container-backend-candidate) +to build and publish the runtime and governance images into the internal +registry. A green PR image build does not publish an image or deploy it. +Record the source SHA and both output digests before configuring the site. +For the shipped data-science notebook, also run the +[notebook image preparation target](container-images.md#prepare-the-data-science-notebook-for-container-workspaces) +before any of the three container registration paths. Its output supplies the +notebook image and scoped pull values used in the profile approval below. + +| Input | Value / consumer | +|---|---| +| `global.targetRevision` | Reviewed site/chart Git commit. The Make checkout must contain the same chart and shared projector code. | +| `containerSites..automaticEnrollment.revision` | Full reviewed Git SHA for controller and composition charts. Use the same field under `portal.containerSites` for RHDH. | +| `automaticEnrollment.image` | Runtime image digest. The projector passes it to enrollment, gate/harness/model preparation, the reconciler and relay membership binding. Do not hand-edit their generated images. | +| `automaticEnrollment.buildNamespace` | Namespace containing the runtime ImageStream; charts grant only the named image's pull access to their service accounts. | +| `stack..revision`, `automaticEnrollment.relay.revision` | Reviewed chart revisions for each component and the browser relay. | +| Governance Application `source.targetRevision` and Helm `image` | Reviewed governance chart revision and governance image digest. Governance is a shared platform dependency, not a per-workspace image override. | + +For an internal governance image, include its pull grant in the platform GitOps +configuration. The current governance Deployment uses the `default` service +account. Replace both namespace placeholders; the grant permits only this +ImageStream. Runtime pull grants are generated by the workspace charts. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: governance-image-pull, namespace: } +rules: + - apiGroups: [image.openshift.io] + resources: [imagestreams/layers] + resourceNames: [governance-interceptor] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: governance-image-pull, namespace: } +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: governance-image-pull} +subjects: [{kind: ServiceAccount, name: default, namespace: }] +``` + +Keep stock gateway, supervisor, OpenClaw, proxy and audit images at their approved +pins unless those components are deliberately being changed. The standalone +`projection.image` is a separate Python helper; its shipped pin does not need to +change merely because the runtime image changed. + +Commit these public pins to the operator's configuration repository, then use +its normal Argo sync or the documented Make entry point. Wait for component +rollouts before capturing the boundary below. Changed helper inputs produce +new immutable Job names. A changed enrolled boundary still fails closed; +replacing its approved hash does not replace the immutable enrollment or +authorize adopting different resource identities. + +## Standalone installation and first-site approval + +Save the complete public configuration as `container-users.yaml`, then: + +```sh +export DEPLOYMENT_MODE=container +export OPENSHELL_SAW_NAME=alice +export SAW_USERS_VALUES_FILE="$PWD/container-users.yaml" +export SAW_INSTALL_NAMESPACE=saw-workspaces +make saw-create +``` + +The helper renders the chart before making changes, checks the target ingress +domain and required APIs, rejects a foreign installation identity and installs +`workspace-alice`. It creates an Argo-managed installation namespace if absent; +it does not relabel an existing namespace. With enrollment disabled the result +is `AwaitingApproval`, **not** a running-workspace claim. Argo reconciles the +foundation while the application sandbox remains absent. + +On a cold static installation, the projector identity Job and ApplicationSet +start asynchronously. A missing plugin Secret error can precede identity +publication; the ApplicationSet then recovers on its automatic retry. Wait for +its `ParametersGenerated=True` condition and healthy foundation Applications +before reviewing the boundary. To verify creation, inspect the generated +Application's `metadata.ownerReferences` in the Argo namespace. Its controller +reference must match the ApplicationSet name **and UID**. Check the child +Application's sync and health independently of its downstream workspace Pods +and PVCs. Record any failed bootstrap attempts separately from the final +installation result. + +Wait for foundation Applications to be Synced/Healthy. Inspect the exact public +objects and normalized boundary separately: + +```sh +oc get applications -n vp-gitops -l openshell.pattern/owner=alice +uv run --locked python scripts/portal-container/inspect-boundary.py \ + --owner alice --registration "$REGISTRATION" \ + --installation-id "$INSTALLATION_ID" \ + --governance-namespace "$GOVERNANCE_NAMESPACE" \ + --api-server "$EXPECTED_API_SERVER" --show-objects > boundary-objects.json +uv run --locked python scripts/portal-container/inspect-boundary.py \ + --owner alice --registration "$REGISTRATION" \ + --installation-id "$INSTALLATION_ID" \ + --governance-namespace "$GOVERNANCE_NAMESPACE" \ + --api-server "$EXPECTED_API_SERVER" > boundary-reference.json +uv run --locked python scripts/portal-container/inspect-boundary.py \ + --owner alice --registration "$REGISTRATION" \ + --installation-id "$INSTALLATION_ID" \ + --governance-namespace "$GOVERNANCE_NAMESPACE" \ + --api-server "$EXPECTED_API_SERVER" --profile-digest +``` + +These commands read public objects and do not grant approval or read Secrets. +The RHDH create pipeline stays pending while the operator reviews a fresh container +registration. Its intent wait uses `portal.sandboxReadyTimeoutMinutes` (at least +10 minutes). Approve within this budget or the request fails. The VM provisioning +stage retains its 10-minute wait. + +Review the resource specifications against the intended configuration. Put the +approved reference under `automaticEnrollment.reference` and enable enrollment +in the values file. Review the rendered BOM and put its digest under +`automaticEnrollment.approvedProfiles.data-science`; foundation mode requires +neither placeholder hashes nor a previously approved profile digest. Re-run +`make saw-create`. For a previously approved site, +enrollment can be enabled on its first install. + +`approvedProfiles` hashes the exact rendered BOM ConfigMap data, including image +overrides and optional harness bundles. Recompute it when effective inputs differ; +a standalone approval applies to RHDH only when their rendered data is identical. +`harnessEnabled` controls optional `harnessRef` bundles. Enrollment attaches the +audited tool gate independently to each supported agent sandbox. + +The projector reads the immutable enrollment and relay publications; no one +copies generated intent or resource UIDs back into Git. Argo then enables the +reconciler, prepared mounts and owner-only browser relay. `make saw-create` +waits for the same current-generation, BOM and browser-readiness conditions as +RHDH. During bootstrap a missing enrollment or relay subject is expected. Both +projectors can read their existing generated Application to distinguish bootstrap +from a lost publication. For the same owner, registration and installation, they +retain the exact prior child parameters if an enrolled binding or published relay +subject disappears. Other declared owners continue to update. An unreadable +child, malformed enrollment or ownership mismatch fails projection. Removal of +a declared static owner remains an intentional prune. When the last static +container entry is removed with projection still enabled, only the empty +projector configuration remains for that backend. Argo can prune its children +without enabling empty-application sync or retaining any projector RBAC. Static sites must sync the +new namespaced Role granting GET on their named children before using the updated +projector. No migration of ApplicationSets or child ownership is required. +If the required Keycloak owner lookup fails, the shared VM/container projection +also stops until lookup recovers. An unavailable identity service must not be +interpreted as a missing owner or an empty membership list. + +```sh +make saw-status # JSON readiness and authenticated UI links +make saw-delete # foreground deletion of this owned registration +``` + +The shared installation namespace and retained audit history are not removed +by per-workspace deletion. Provider material provisioned directly in Vault is +operator-owned; deleting the standalone Helm release does not erase it. + +## Pattern and RHDH enablement + +For static pattern users, put the same values under `overrides/saw-users.yaml`, +enable `projection`, commit them and let the existing `saw-users` Application +reconcile. Its namespace must be separate from Argo CD. VM users keep their +existing Applications and owner-resolution behavior. A container user's +resolved `ownerSubject` does not enable an owner override for VM users. + +For RHDH, use `portal.containerSitesSecretName` when the site includes owner +subjects or cluster resource identifiers and the pattern repository is public. +Create a Secret in `saw-portal` with a `container-sites.json` key. Its JSON +value is the same owner-to-site map used by `portal.containerSites`. Set only +the Secret name in the cluster's values file. The create pipeline checks the +owner and selected profile against this map. The form shows catalog profiles +for each enabled site mode, so an unapproved owner can see a choice but cannot +register it. + +```yaml +clusterGroup: + applications: + openshell-rhdh: + overrides: + - name: portal.containerSitesSecretName + value: saw-portal-container-sites + forceString: true +``` + +### Approve the first RHDH container profile + +1. **Review the profile before offering it.** From the reviewed chart revision, + render the public BOM with the effective inputs for this owner. Set + `profiles: [data-science]` in `bom-values.yaml`. Include its effective + `imageOverrides`, `webSearch.enabled` and `harnessEnabled`, matching the + intended workspace. Inspect the rendered providers, models, sandbox images + and harness files. Hash the ConfigMap's `data` with the same function the + live verifier uses. + + ```sh + helm template review charts/saw-bom -f bom-values.yaml \ + --show-only templates/configmap-bom.yaml > rendered-bom.yaml + less rendered-bom.yaml + PROFILE_DIGEST=$(PYTHONPATH=kubernetes-runtime uv run --locked python -c \ + 'import yaml; from saw_kubernetes.boundary import fingerprint; print(fingerprint(yaml.safe_load(open("rendered-bom.yaml"))["data"]))') + printf '%s\n' "$PROFILE_DIGEST" + ``` + +2. **Offer the approved profile through GitOps.** Merge the following fields + into the owner's complete site in `overrides/saw-users.yaml`. Replace the + two placeholders with the resolved Keycloak subject and the printed 64-hex + digest. Keep the existing component, relay and policy settings. + + ```yaml + portal: + deploymentModes: [vm, container] + pruneOnRemove: true + containerSites: + alice: + subject: + automaticEnrollment: + enabled: false + approvedProfiles: + data-science: + ``` + + Commit and push the site change. Wait for `openshell-rhdh` to sync that + revision and for the Developer Hub Deployment to finish rolling out. Sign + in as the owner, open **Create or update my agent workspace**, choose + **Container** and **data-science**, supply the profile's requested + credentials in the form, and submit. Argo generates a child Application + after the portal's `register` task succeeds. The operator can then inspect + the new registration while the pipeline waits for readiness. + +3. **Verify the live BOM and approve enrollment.** For a portal registration + the registry ConfigMap supplies the registration ID. The inspection tool + derives the installation UUID. Set `OWNER`, `GOVERNANCE_NAMESPACE` and + the expected cluster API URL before running: + + ```sh + REGISTRATION=$(oc -n saw-portal get cm "saw-ws-$OWNER" -o json | + jq -er '.data["user.json"] | fromjson | .vaultPrefix | split("/") | last | select(test("^[0-9a-f]{16}$"))') + inspect_owner() { + uv run --locked python scripts/portal-container/inspect-boundary.py \ + --owner "$OWNER" --registration "$REGISTRATION" \ + --governance-namespace "$GOVERNANCE_NAMESPACE" \ + --api-server "$EXPECTED_API_SERVER" "$@" + } + LIVE_DIGEST=$(inspect_owner --profile-digest) && test "$LIVE_DIGEST" = "$PROFILE_DIGEST" + ``` + + Continue only after the comparison succeeds. If it differs, review the + effective inputs and the live BOM. Run `inspect_owner --show-objects` and + review those specifications, then run `inspect_owner > boundary-reference.json`. + Put that reviewed reference in the **same owner's** `automaticEnrollment.reference`, + change `enabled` to `true` and commit the change to site Git within the + configured approval timeout. Verify the create task reaches ready and open + the owner-only chat. The audit and gate claims must be Bound on an + RWX-capable filesystem for the audited workspace to run. + +Keep `projection.enabled` off for the RHDH-generated child values. The portal +performs that projection. The Delete form and state actions use the backend +recorded in the registration. + +## Updates and current limitations + +Code rollout can be pinned per owner without changing their registration. The +operator sets `workspaceRevisions` in `saw-users` values (static/Make), or the same +map under `sawUsers.workspaceRevisions` in the RHDH chart: + +```yaml +sawUsers: # RHDH values; omit this outer key for static/Make values + workspaceRevisions: + existing-owner: + candidate-owner: +``` + +Each pin must be a full lowercase Git commit SHA in the configured +`global.repoURL`. Unlisted owners continue using `global.targetRevision`. The +shared projector selects the revision for both the owner Application and its +children, for VM and container registrations. Direct static VM Applications +honor the same map. Forms and registration records cannot set these pins or +select a different repository. A recreated registration uses the operator's +current pin for that owner. + +When chart and values live in the same Git repository, keep them on one Argo +source revision. Argo rejects different revisions of that repository for the +chart and a `$values` reference. Integrate the reviewed portal chart into the site +revision, or publish it as a separately versioned Helm artifact. Owner pins still +control the generated workload revisions. + +Both RHDH and static projector parent Applications preserve Argo's two +controller-owned `pre-delete-finalizer.argocd.argoproj.io` finalizers (plain and +`/cleanup`) during comparison and sync. The exact-value JQ rule plus +`RespectIgnoreDifferences=true` prevents repeated parent self-heal from removing +hook state. Resource-deletion finalizers, source revisions and workload specs +remain managed and checked; this does not disable deletion hooks or pruning. + +Pin existing owners before upgrading shared portal code when their child charts +must stay unchanged. The portal's chart revision and its generated workload +revisions are separate: retain the site's `global.targetRevision` fallback and +use the map to move only reviewed owners. A source pin does not approve a new +BOM, image, boundary or storage identity; those retain their normal checks. + +Update public `modelInputs`, increment `intentGeneration` and +`restartGeneration`, then reapply through the same entry point. Keep +`initialModelInputs` and registration identity fixed. For stop/start, change +`desiredState` and increment `intentGeneration`; storage remains attached to the +registration. Do not switch an enrolled workspace back to foundation mode. + +There is no additional upgrade controller. Argo reconciles pinned chart/image +changes; immutable preparation Jobs get input-derived revision names. An +enrolled boundary change requires deliberate approval, not automatic trust in +a newly observed hash. VM installation keeps its current signed installer, +live-input reconciliation and reboot path. + +Administrator sessions are created per operation using the shared VM mechanism. +Their private material stays in memory-backed volumes; the charts grant the +controller and membership Job access only to their named public client-trust +Secret. No retained admin key, `adminDays` setting, renewal controller or operator +credential injection is needed. Server TLS and retained gateway identity remain +separate from this mechanism; a normal resync never rotates that identity. + +VM guest commands +(`saw-vm-ssh`, installer logs and VM restart) remain VM-only. Container logs come +from the reconciler in `saw-ctl-` and the application Pods; lifecycle +changes use public intent or RHDH. See the architecture page's +[current boundaries](container-workspaces.md#current-boundaries) for the remaining +security and access limitations. + +To reproduce the Pod admission regression on an enrolled container workspace, +use a kubeconfig authorized to impersonate its gateway and Agent Sandbox +controller. This requires running workload and supervisor Pods. The probes use +API-server dry-run, including a malicious template submitted as the legitimate +controller and an attempt to remove the workload’s role during an update. + +```sh +python3 scripts/portal-container/check-pod-admission.py \ + --kubeconfig "$KUBECONFIG" --namespace "$RUNTIME_NAMESPACE" --gateway "$GATEWAY" +``` + +Each probe must report denial by the workspace Pod policy. An RBAC error, +missing API, or collision with an existing Pod fails the check. + +The Task reads the Tekton-owned `tekton.dev/pipelineRun` Pod label through the +downward API to identify its credential reservation. If a pending writer disappears, inspect its PipelineRun and the owned +Vault generation before clearing the writer annotation. A missing holder fails +closed. A terminated failed run can be retried for the same registration. + + +### Interrupted creates and updates + +RHDH reserves the registration before writing credentials. The catalog keeps +the owner component visible during publication. A terminal writer with an +incomplete reservation displays retry/delete instructions. A missing writer +requires operator diagnosis. On an interrupted update, GitOps continues to +project the last committed public configuration. Retry the create form with +all required inputs to finish the same registration. Vault writes to multiple +secrets are independent; an earlier write may already be visible to ESO. + +The reconciler records the ID from a successful sandbox create before waiting +for readiness. It replays an interrupted create with the original request ID +on a Running pass. If shutdown encounters an outstanding create with no proven +receipt, its reason is `StartToRecoverSandboxReceipt`. Request start to recover +the receipt, then stop. Shutdown will never create a missing sandbox. + +The intent CLI accepts `--restart-generation`. When updating with +`--previous-intent`, omitting the option preserves the previous value, +including absence on legacy intents. The value ranges from zero through the +intent generation and cannot decrease. + +### Discover and review container egress + +The optional container topology preflight reuses the VM's API Service and ready +EndpointSlice discovery. It also reads the OpenShift DNS Service and endpoint +ports. It runs HTTPS discovery from an existing, administrator-selected Pod to +find the issuer addresses in the cluster's DNS view. Use a trusted Pod with +Python 3, the correct CA trust, DNS and HTTPS access. The caller needs read +access to those cluster resources and exec access to that Pod. The probe sends +no credentials, follows no redirects, and verifies the issuer hostname and +discovery document at every IPv4 answer. IPv6 issuer answers are unsupported. + +Keep governance, proxy and other administrator-approved rules in your base site +file. When migrating an existing site, remove the hand-maintained API, issuer +and DNS rules from the base lists first. Every base rule is preserved. A stale +address left in the base remains allowed. Configure the gateway or relay issuer +once, or omit both to discover the Keycloak host in the relay's configured +namespace with the same helper used by VM Make. The default realm is +`openshell`; set `KEYCLOAK_REALM` for another realm. Explicit gateway and relay +issuer URLs must agree. + +```sh +export SAW_USERS_VALUES_FILE="$PWD/container-users.yaml" # base values +# On a stock SAW platform, the portal generator has Python 3 and access to its issuer. +PROBE_POD=$(oc -n saw-portal get pods -l app.kubernetes.io/name=saw-workspaces-generator -o json | + jq -er '[.items[] | select(.metadata.deletionTimestamp == null and + any(.status.conditions[]?; .type == "Ready" and .status == "True"))][0].metadata.name') +export SAW_TOPOLOGY_PROBE="saw-portal/$PROBE_POD" +export SAW_TOPOLOGY_CONTAINER=generator +export SAW_TOPOLOGY_OVERLAY="$PWD/container-network.yaml" +# Set SAW_TOPOLOGY_CONTAINER if the Pod has multiple containers. +make container-topology > "$SAW_TOPOLOGY_OVERLAY.new" && + mv "$SAW_TOPOLOGY_OVERLAY.new" "$SAW_TOPOLOGY_OVERLAY" +make check-container-topology +``` + +For GitOps, commit the generated file and include it **after** the base file +in the Application's Helm value files. Generate from `containerSites` for +saw-users or `portal.containerSites` for RHDH. A base file containing both +produces both sets of overrides. Run the check against the original base file +and the committed overlay before enrollment and when DNS or API endpoints +change. A mismatch exits nonzero. Review the regenerated diff before syncing. + +For a Pattern site with both sources, store the reviewed overlay at +`overrides/container-network.yaml` and declare the value file order in the +site's cluster-group values. Preserve any other value files the site already +uses. + +```yaml +clusterGroup: + applications: + saw-users: + extraValueFiles: [/overrides/saw-users.yaml, /overrides/container-network.yaml] + openshell-rhdh: + extraValueFiles: [/overrides/saw-users.yaml, /overrides/container-network.yaml] +``` + +For standalone Make, keep the base and overlay paths for the installation. +Run the Ready Pod selection above again before each `make saw-create`, +including enrollment after the boundary review. A portal rollout can replace +the probe Pod between those steps. Check the **existing** overlay against +the current trusted Pod. Regenerate it only after reviewing a topology change. +On first +foundation apply (`automaticEnrollment.enabled: false`), Make can create the +specified overlay if absent. Later applies require the saved values to match +live discovery. Status and delete can use that file without contacting the +issuer. Keep the file per installation. Existing deployments using explicit +egress inputs can continue to use that workflow. + +Generated rules grant individual addresses and ports to the gateway and relay. +API and DNS rules also go to bootstrap and, for static sites, the projector. +Workload/supervisor policies, provider host approvals, storage and enrollment +approval still come from their existing sources. This preflight validates the +probe Pod's view; workload readiness after deployment verifies the actual +gateway path. diff --git a/docs/container-runtime.md b/docs/container-runtime.md index 8ff8f4898..7efb5e2f3 100644 --- a/docs/container-runtime.md +++ b/docs/container-runtime.md @@ -1,5 +1,44 @@ # Container Runtime Support: Docker and Podman +## Shared browser transport + +VM and Kubernetes workspaces package the same +[`saw_ui_limit.py`](../charts/openshell-saw/files/installer/saw_ui_limit.py) +connection limiter. The VM embeds it in the signed installer; the independent +relay chart receives a generated byte-identical copy, checked by +`scripts/saw-profile-catalog.py --check`. + +For Kubernetes workspaces the request path is: + +```mermaid +flowchart LR + Browser --> Route --> Login[Owner-only oauth2-proxy :4180] + Login --> Transport[Shared transport 127.0.0.1:18789] + Transport --> CLI[Stock OpenShell forward 127.0.0.1:28789] + CLI --> Gateway --> App[Sandbox OpenClaw :18789] +``` + +The transport is a restricted Python container with only a read-only script +ConfigMap: no client secret, CLI session or Kubernetes token mount. It allows +16 upstream connections, reserving capacity below OpenShell's sandbox limit. +It starts Python with `-I` so the adjacent OIDC `token.py` helper cannot shadow +Python's standard-library `token` module during transport startup. +While the CLI restarts after an expired authentication token, the stable listener +queues connections with bounded retries **before sending any request bytes**. +The container's `--connect-only` mode never replays an HTTP request or WebSocket +frame, including a GET whose response was lost. VM transport behavior remains +unchanged. + +Refreshing the CLI's on-disk token does not refresh an already running forward +session in the pinned stock release. Established WebSockets can therefore close +at expiry; clients must reconnect through their existing authenticated browser +session. The transport masks short connection-refused gaps for new connections, +not a failed token exchange or an indefinitely unavailable gateway. Qualification +must cross token expiry and verify reconnect, owner isolation and no duplicate +tool execution; passing a short chat does not establish that coverage. + +## VM container runtimes + The gateway VM supports two container runtimes, selectable at deploy time via a single Helm value. No golden image rebuild is required to switch — both images are pre-built and available. diff --git a/docs/container-workspaces.md b/docs/container-workspaces.md new file mode 100644 index 000000000..d6d5bfb18 --- /dev/null +++ b/docs/container-workspaces.md @@ -0,0 +1,367 @@ +# Container workspaces + +Container mode adds a Kubernetes backend to SAW's existing workspace lifecycle. +VM remains the default. Both backends consume the same SAW-BOM language, profile +catalog, provider profiles and tool-action policy. Container mode uses the stock +OpenShell Kubernetes driver; it does not add a workspace CRD or modify that driver. +See [installation and enablement](container-installation.md) for the pattern, +standalone Make and RHDH entry points, inputs and operational boundaries. + +## Architecture and ownership + +```mermaid +flowchart TD + Profiles[SAW-BOM profiles in Git] --> Catalog[Generated catalog and RHDH forms] + Profiles --> BOM[saw-bom: profile ConfigMap] + RHDH[RHDH owner request] --> Pipeline[Shared create/update/delete pipelines] + Static[Operator YAML: pattern or Make] --> Projection + Pipeline --> Vault[Vault: registration-scoped credentials] + Pipeline --> Registry[Workspace registration ConfigMap] + Registry --> Projection[Common workspace projector] + Catalog --> Projection + Enrollment[Immutable enrollment publication] --> Projection + Projection --> Argo[ApplicationSet / Argo CD] + Argo --> BOM + Argo --> VM[VM chart / in-guest apply_bom] + Argo --> Container[Container composition charts] + BOM --> Core[Shared SAW engine] + Core --> VM + Core --> Reconciler[Kubernetes reconciler] + Container --> Reconciler + Reconciler -->|operator mTLS RPC| Gateway[Stock OpenShell gateway] + Gateway -->|Kubernetes driver| Supervisor[Supervisor Pod] + Gateway --> Sandbox[agent-sandbox resource] + Sandbox --> Agent[Application Pod] + Supervisor <--> Agent +``` + +| Component | Responsibility and boundary | +|---|---| +| `saw-bom` | Publishes the same profile documents for either backend. A profile describes workspaces, providers, sandboxes, harnesses and inference inputs. | +| Shared `saw_core.py` | Owns profile parsing, validation, dependency ordering, provider selection, typed inference inputs and tool-gate bundle construction. VM and container adapters load the same source. | +| Shared `saw_admin.py` | Creates a temporary administrator identity for an operation and removes its private files afterward. The VM publishes temporary trust through its local file; the Kubernetes adapter updates one named public trust Secret. Both use the same certificate issuance and cleanup code. | +| VM `apply_bom.py` | Retains VM installation, Podman, guest files and VM-specific lifecycle operations. The signed installer embeds the shared engine bytes. | +| Common `workspace.py` | Projects a registration and catalog into `saw-users` values: sandbox topology, image overrides, approved profile digest, model inputs and public enrollment. Also implements shared readiness: matching BOM/intent generation, healthy Argo application and owner login challenge. It performs no credential storage or resource mutation. | +| RHDH adapter | Authenticates requests, resolves owners, stores provider inputs, maintains registration and drives the existing Tekton/ApplicationSet flow. Its forms come from the shared catalog. | +| Standalone projector | An ApplicationSet plugin installed by `saw-users` when `projection.enabled`. Uses the same `workspace.py` as RHDH, with operator-owned YAML instead of portal registrations. Reads named public enrollment ConfigMaps and its own named child Applications. It has no Kubernetes write or Secret-read permission. | +| Container composition | Creates per-registration identity, network policy, credential proxy, gateway, PVCs, audit services, preparation Jobs, reconciler and browser relay through Argo CD. | +| Enrollment Job | Compares deployed resources against an administrator-approved boundary; records actual resource UIDs and hashes, PVC identities and public trust material in an immutable publication. It cannot approve an arbitrary observed configuration. | +| Kubernetes reconciler | Reads BOM + intent, checks the approved boundary and attachment identities, compiles stock OpenShell RPCs, and reconciles owned objects using a retained ledger. It does not create application Pods directly. | +| Stock gateway / driver | Implements workspace/provider/sandbox APIs and creates supervisor Pods and agent-sandbox resources. Gateway state has its own PVC. | +| Stock agent-sandbox controller | Reconciles the workload resource into the application Pod. Its CRD/controller is a separate prerequisite, not a SAW API. | +| Governance interceptor | Supplies signed provider profiles and policy. SAW's extension restricts Kubernetes driver configuration to an approved PVC/resource envelope submitted by a platform mTLS operator. | +| Credential proxy | Resolves opaque credential markers using trusted-namespace Secrets and the shared provider policy. It supplies credentials only for approved destinations. | +| Audit writer | Owns retained history outside the application namespace and signs acknowledgements after append. The application mounts an exchange volume, not the history PVC. | +| Browser relay | Combines owner-only login, a dedicated nonadmin Keycloak client, shared connection limiter and stock OpenShell forwarding. Its membership is scoped to the intended workspace. Only its membership Job opens a temporary gateway operator session; the UI has no operator identity or trust-write permission. | + +The implementation lives in `kubernetes-runtime/` and the `saw-kubernetes-*`, +`openshell-saw-kubernetes`, `saw-runtime-policies`, `saw-audit-writer`, +`saw-workspace-relay` and `saw-portal-container` charts. The last chart composes +existing component charts for both installation paths; its name does not imply +a dependency on Developer Hub or a second BOM dialect. + +Each registration has a runtime namespace (`saw-`), a separate control +namespace (`saw-ctl-`), retained history +(`saw-history-`) and a browser namespace per UI sandbox. Gateway +administrator keys exist only in the controller or membership Job's memory-backed +volume in the control namespace. Controller state and provider Secrets stay in +that namespace. Gateway and supervisor have different service accounts and +network permissions from the application. Each workspace has its own gateway; +governance, Keycloak, Vault, Argo and agent-sandbox are shared platform services. + +Provider creation is restart-safe: the ledger records a pending payload digest +and original generation before `CreateProvider`. After a lost response, the +reconciler reuses the request ID and accepts the stock gateway's durable receipt +only for the unchanged resource revision. It never infers credential ownership +from a redacted read. See [recovery and qualification](provider-driver-qualification.md). + +The briefing harness mounts two subdirectories from existing approved PVCs: +the digest-named harness on the read-only gate PVC at `/sandbox/harness`, and +`briefing` on the writable workspace PVC at `/sandbox/briefing`. Governance +admits only these exact aliases in addition to the base mounts; it does not +allow arbitrary additional mounts, subpaths or volume sources. + +## Configuration and policy distribution + +There are two different BOMs: the **installer BOM** pins VM runtime components; +the **SAW-BOM profiles** describe the user's workloads. Sharing the latter does +not mean that a container runs the VM installer. + +1. An operator commits approved profiles, component image digests, provider + destinations and site topology. `make catalog` generates forms and catalogs, + and mirrors canonical policy/projection sources into their packaging charts. +2. RHDH accepts the deployment mode and a profile allowed for that backend. The + owner cannot supply a RuntimeClass, PVC name, executable image or policy + override. Provider credentials go through the request Secret into Vault; + registration stores references and permitted public metadata. + A static installation supplies the same public registration and site values + directly. Its operator provisions provider material in Vault; the projector + never handles those credentials. +3. `saw-users` selects the backend. VM omission remains `vm`; a configured + container entry selects the composition chart. An existing registration's + backend cannot be changed implicitly by changing a chart default. +4. Argo creates the foundation. Enrollment verifies the approved configuration + and publishes the registration binding. The generator reads only the named + public publications and reprojects the values to enable runtime resources. + A new site can set `automaticEnrollment.enabled: false` to deploy its + foundation for review. Enabling enrollment requires approved reference + hashes; discovery is not approval. +5. The reconciler requires a coherent BOM digest and intent generation, checks + namespace/PVC UIDs and the live boundary, and then calls the gateway. It + refuses stale, substituted or unsupported inputs rather than falling back to + another backend. + +```mermaid +sequenceDiagram + participant Git as Operator / Git + participant Argo as Argo CD + participant Enroll as Enrollment Job + participant Project as Workspace projection + participant Runtime as Reconciler + participant Gateway as OpenShell gateway + participant Pod as Workload + Git->>Argo: Approved profile + site + desired registration + Argo->>Argo: Identity, network, proxy, gateway, PVCs, BOM + Argo->>Enroll: Pinned bootstrap inputs + Enroll->>Enroll: Check approved boundary and ownership + Enroll->>Project: Immutable binding: UIDs, hashes, public keys + Project->>Argo: Intent, preparation, controller, relay values + Argo->>Runtime: BOM + intent + public gateway CA + ledger + Runtime->>Runtime: Compile shared BOM; verify current boundary + Runtime->>Gateway: Establish temporary operator mTLS trust + Runtime->>Gateway: Reconcile owned workspace/providers/sandboxes + Runtime->>Gateway: Withdraw temporary operator trust + Gateway->>Pod: Stock Kubernetes driver and agent-sandbox controller +``` + +Policies protect different boundaries: + +- **Governance policy/provider profiles** control OpenShell operations, provider + endpoints and the permitted driver-config envelope. The SAW extension is + maintained as ordinary Rust plus an explicit patch to the pinned interceptor + example. The existing OpenShell v0.1.2 provider-annotation ordering workaround + remains; this change does not claim an entirely unmodified interceptor. +- **OpenShift admission and network policy** constrain Pod roles, service + accounts, placement and network reachability. A reconciler's permission to + call the gateway does not bypass these controls. +- **Credential-proxy policy** derives from the same provider catalog and Helm + helper used by VM mode; credentials are not copied into application Pods. +- **Tool-action policy** uses the same `tool-actions.yaml`, decision engine and + OpenClaw hook as VM mode. The Kubernetes adapter changes storage and audit + transport, not the decision rules. + +### Administrator sessions and retained identity + +The controller and relay membership Job use a per-operation mTLS operator +identity. There is no retained administrator certificate to renew and no new +certificate service. Each operation creates a one-day P-256 CA and an +`OU=operator` client certificate in a memory-backed `emptyDir`. The Kubernetes +gateway enables OpenShell operator authentication and disables ordinary mTLS +user authentication. OIDC remains the user login path. The CA private key is +destroyed immediately after signing; the client key is removed when the +operation exits. + +Enrollment also records the selected gateway version in the approved profile +plan. Set `site.stack.gateway.values.kubernetes.gateway.version` when the site +overrides the gateway release. Keep it aligned with the gateway image. The +portal chart default matches the pinned gateway chart version. A mismatch blocks +runtime plan verification. + +```mermaid +sequenceDiagram + participant Client as Controller / membership Job + participant Shared as Shared admin-session code + participant API as Kubernetes API + participant Gateway as Stock gateway + Client->>API: Wait for this exact Pod UID to become Running + Client->>Shared: Create temporary identity in tmpfs + Shared-->>Client: Operator key/certificate + public session CA + Client->>API: CAS update named client-ca Secret: add public CA + API-->>Gateway: Project updated ca.crt volume + Gateway->>Gateway: Reload TLS trust + Client->>Gateway: Wait for authenticated operator RPC; perform operation + Client->>Gateway: Close client channel + Client->>API: CAS update: remove this session CA + Client->>Shared: Remove private session files +``` + +The Kubernetes adapter retains concurrent sessions using `resourceVersion` +compare-and-swap. Each entry is bound to its originating Pod UID and certificate +expiry. A per-Pod file lock serializes operations. After a container restart in +the same Pod, the next operation withdraws that Pod's abandoned entries and +removes its old private files from `emptyDir` before issuing a new identity. +At startup, the process can precede kubelet's published `Running` phase. The +adapter waits up to 30 seconds while its exact Pod UID is `Pending`, then issues +the identity only after observing `Running`. Missing, replaced, deleting or +terminal Pods and API failures stop the operation. Publication still rechecks +the holder; this wait does not authorize trust for a Pending Pod. +Private-file cleanup also runs when trust recovery cannot reach the API. +Updates prune expired sessions and sessions whose Pod has exited or been +replaced. If cleanup loses API access, the next successful trust update removes +the abandoned entry; the certificate's one-day validity also bounds acceptance +on new connections. Secret volume propagation is asynchronous: publication is +followed by an authenticated probe, not an assumed immediate reload. Withdrawal +closes the caller's connection and removes its key; it is not instantaneous +revocation of connections already held by another process. + +Only the controller and membership Job can read/update the named public +`-client-ca` Secret. They can read Pod metadata in their trusted namespace +to verify session holders. These permissions confer gateway operator authority +and are not granted to application or browser service accounts. The +bootstrap Job preserves active sessions when rechecking identity projections. + +Gateway server TLS, JWT signing keys and the credential-encryption key remain in +the create-once retained identity bundle; opening an admin session does not rotate +them, alter workspace IDs, or interrupt the gateway. The public server CA is +projected separately as an immutable ConfigMap. The VM embeds both shared modules +inside its existing signed `apply_bom.py`, preserving the signed manifest format. + +## Inference, tools and browser data paths + +```mermaid +flowchart LR + Browser[Owner browser] --> Login[Owner-only OIDC login] + Login --> Relay[Scoped relay / OpenShell forward] + Relay --> Gateway[Gateway and supervisor] + Gateway --> Agent[Agent / OpenClaw] + Agent -->|Provider marker| Proxy[Trusted credential proxy] + Vault[Vault] --> ESO[External Secrets] + ESO --> Secrets[Trusted-namespace Secrets] + Secrets --> Proxy + Proxy -->|Real credential| Provider[Approved inference / tool API] + Agent --> Gate[Shared tool gate] + Gate --> Exchange[Audit exchange PVC] + Exchange --> Writer[Restricted forwarder and writer] + Writer --> History[Retained history PVC] + Writer -->|Signed append receipt| Gate + Policy[Prepared policy / identity / harness] -->|Read-only mount| Agent +``` + +Model selection is public configuration; API keys and refresh material are not. +The compiler resolves an explicitly attached provider and an approved endpoint. +OpenAI-compatible agent inference currently supports NVIDIA, OpenAI and the +approved custom-inference path; unsupported combinations fail explicitly. + +The container startup publishes whole managed OpenClaw configuration subtrees +through stock `openclaw config set --batch-file`, the same native writer used +by VM configuration commands. It preserves OpenClaw's metadata and backup +bookkeeping; directly replacing `openclaw.json` can trigger recovery of an older +model selection. Managed gateway, provider, agent and tool/plugin settings are +replaced together so removed permissions or providers do not survive an update. + +An explicit container model uses an internal OpenClaw provider ID such as +`saw-openai/gpt-4.1`. This prevents the stock built-in model catalog from +rewriting a requested model to a different release. The BOM still names +`openai`; its approved OpenAI endpoint, protocol, credential marker and proxy +authorization are unchanged. Chat transcripts therefore identify the internal +provider alias and the exact requested model. VM native onboarding remains +unchanged. The native writer regression can run against a stock binary with +`node tests/openclaw-config-publication.mjs`; add `--expect-restore` on versions +that restore configurations missing metadata. It uses synthetic credentials in +temporary state and sends no inference requests. + +The application receives its writable workspace, a read-only prepared gate and +optional harness, and an audit exchange. The writer's history and signing key +remain outside its mounts. A missing/invalid acknowledgement blocks the gated +operation. This is an application-hook control and an append receipt, not proof +that arbitrary code inside a compromised application cannot perform other +operations. Do not equate a recorded approval request with a completed tool +execution or a recorded final human decision. + +The browser authenticates with the pinned issuer and enrolled subject. The +authenticated relay replaces any caller-supplied relay identity with a random +slot-specific proof. OpenClaw checks the proof as well as the subject. The +preparation Job publishes it into that slot's read-only gate volume. A direct +OpenShell forward with a guessed username or copied subject lacks this proof. +Processes already executing as the agent can read their own local gateway +configuration and remain inside the application's trust boundary. + +OpenClaw treats `onResolution` as a notification and catches its exceptions. +The signed container gate therefore exits the executor synchronously if an +approved decision cannot obtain its receipt, before control returns to the +tool. The exchange request is fsynced before publication and the forwarder +retries it. An audit outage can interrupt the agent and require a workspace +restart. Retain the original decision and verify the history before retrying +a consequential action. + +## Lifecycle and installation surface + +The portal reserves the registry generation with a resource-version check +before writing provider credentials. Pending entries do not project new +workspaces. A failed publisher leaves its generation recorded for recovery. +A later request can resume that same generation after the old pipeline ends. +Deletion destroys credentials only for the registered generation. + + +RHDH uses the existing create/update/delete forms and pipelines, with a backend +selector and backend-approved profile choices. The registration carries the +backend so status, readiness, restart and cleanup select the matching adapter. +Model changes use versioned public input publication; stop/start retains +workspace storage. Foreground deletion removes owned compute and access +resources while audit history is retained. Old registrations cannot claim a new +registration's resources or credentials. + +Preparation/enrollment/relay Jobs have input-derived names and immutable script +ConfigMaps. A changed image or input creates a new Job rather than editing an +immutable PodSpec. Changing an enrolled security boundary still requires explicit +review; a new Job name does not authorize adopting new identities or overwriting +an existing trusted bundle. + +| Entry point | Current boundary | +|---|---| +| Existing VM pattern and Make quickstart | Retain their existing installation path and default behavior. | +| RHDH container lifecycle | Uses the shared catalog, requests, pipelines and workspace projector; requires an operator-provided container site and approved profile boundary. | +| Static `saw-users` container values | Set `projection.enabled: true`. A read-only ApplicationSet plugin incorporates enrollment and relay publications continuously. VM entries keep their existing root-chart path. | +| Container Make entry point | `DEPLOYMENT_MODE=container` dispatches create/status/delete to the same `saw-users` installation. Argo owns its component applications. Requires the shared platform services and site inputs. | + +RHDH container enablement requires `portal.deploymentModes: [vm, container]` and a +`portal.containerSites.` entry with the resolved subject, approved profile +digests, component revisions/images, network/storage inputs, reviewed boundary +references, and relay identity settings. Provider keys remain in Vault, with +External Secrets projections into the trusted namespace. The site must provide +both RWO storage and a filesystem class suitable for the shared preparation and +audit exchange mounts. Enabling the selector alone does not provide these inputs. + +The runtime image packages the reconciler, pinned OpenShell protobuf SDK, +preparation helpers and canonical shared sources. The existing image publication +actions build it on amd64/arm64 and publish on `main`; sites consume the resulting +digest. For an operator build using an explicitly selected registry: + +```sh +podman build --ignorefile kubernetes-runtime/Containerfile.dockerignore \ + -f kubernetes-runtime/Containerfile \ + -t "$REGISTRY/saw-kubernetes-runtime:$REVISION" . +podman push "$REGISTRY/saw-kubernetes-runtime:$REVISION" +``` + +The build context allowlist excludes local credentials and unrelated files. +Resolve and pin the pushed manifest digest in the site inputs; the build commands +alone do not install the component stack. + +## Current boundaries + +- Container installation adds OpenShift GitOps/ApplicationSet, Agent Sandbox, + native admission and enforced network-policy requirements. The Make command + checks platform APIs; it does not install, replace or adopt shared operators. +- A profile is enabled only with an owner-specific approved boundary and pinned + BOM digest. Adding a BOM does not automatically approve its images, provider + endpoints or storage permissions. One approved profile may contain several + sandboxes; a registration selects one profile and one deployment backend. +- Source and image updates use ordinary Argo reconciliation; there is no second + upgrade manager. Changes to an enrolled boundary or trusted bundle do not gain + approval merely because a new image or Job revision exists. +- The browser path is owner authenticated. External gateway CLI access requires + an explicitly configured passthrough Route and matching server certificate + SAN. VM guest-management commands do not operate on container workspaces. +- An application-level tool hook and signed audit receipts are not kernel-level + confinement. Retained history is isolated from the workload; cluster and + storage administrators remain trusted. + + +Catalog refreshes use a bounded pool for route probes. The catalog can show a +route as pending while its first probe runs. Results expire after 30 seconds +and are scoped to the owner, registration, intent generation, issuer and URL. +Owner readiness checks still probe the live route. For dedicated inference +endpoints, `dedicatedEndpointSecrets` in the generated profile catalog lists +the BOM inputs accepted by the container form and Vault writer. VM forms retain +their profile’s full credential set. diff --git a/docs/kata-workspaces.md b/docs/kata-workspaces.md new file mode 100644 index 000000000..cc87a4f31 --- /dev/null +++ b/docs/kata-workspaces.md @@ -0,0 +1,370 @@ +# Kata workspace qualification + +Kata is under private qualification. The application workload Pod uses the +stock OpenShell Kubernetes driver and a local Kata RuntimeClass. The gateway +and separate supervisor run as ordinary Pods. Each data-science slot is a +separate workload Pod. The workspace BOM, images, storage, credential routing, +relay and audit flows are shared with container workspaces. + +## Qualification status + +On 2026-11-17, one RHDH data-science Kata canary completed. Both workload +Pods were Ready with the Kata RuntimeClass. Each Pod mapped to a different +host QEMU process. The gateway and both supervisors used the default runtime. +Both owner chat links returned fresh model replies. This result does not yet +cover tool allow and deny rules, network denial, secret isolation, audit, +persistence, restart, stop/start, deletion, GitOps, Make, a container +regression, or two active owners. Keep the PR in draft until these checks pass. + +## Check worker KVM access before installing OSC + +Run this from a platform administrator's existing OpenShift login. Supply the +expected API address independently and a selector for the intended worker +pool. The command reads host sysfs through the existing machine-config daemon +on each selected worker. It creates no Pods, operators or machine configs. + +```sh +KATA_API_SERVER=https://api.example.com:6443 \ +KATA_NODE_SELECTOR='saw.example.com/kata-pool=true' make kata-worker-check +``` + +Each selected node must report either `UEFI KVM` or `legacy-BIOS KVM`. An +empty selector, a control-plane node, a missing host probe or missing KVM +prevents a pass. The firmware value is context only. Red Hat's OSC guide lists +CPU virtualization flags and KVM modules for node eligibility; it does not +list UEFI as a requirement. This check does not replace OSC node eligibility +checks or a live workload test. +The running Kata handler, guest seccomp behavior, RWX mounts and the OpenShell +supervisor connection require live qualification. Configure eligible workers +before installing OSC. Creating a KataConfig can reboot workers. +[Red Hat OSC node eligibility checks](https://docs.redhat.com/en/documentation/openshift_sandboxed_containers/1.12/html/deploying_openshift_sandboxed_containers_on_bare-metal_servers/install-osc-overview_metal-osc#install-osc-node-eligibility-checks_metal-osc) + +## Prepare OpenShift sandboxed containers + +The qualification cluster installed the Red Hat OpenShift sandboxed containers +Operator from OperatorHub in `openshift-sandboxed-containers-operator`. A GitOps +installation uses a Namespace, an OperatorGroup scoped to that namespace, and a +Subscription from `redhat-operators`. Select the channel supported by the +cluster. Keep these platform resources outside the SAW workspace charts. Check +that the Subscription has an installed CSV and that the CSV is `Succeeded`. +See the [Red Hat installation procedure](https://docs.redhat.com/en/documentation/openshift_sandboxed_containers/1.12/html/deploying_openshift_sandboxed_containers_on_bare-metal_servers/install-osc-overview_metal-osc). + +Install the local Kata runtime on an approved worker pool with a platform-owned +`KataConfig`. This example uses the same pool label as the preflight command. +Use the selected cluster's supported OSC version and node eligibility rules. + +```yaml +apiVersion: kataconfiguration.openshift.io/v1 +kind: KataConfig +metadata: + name: saw-kata-pool +spec: + enablePeerPods: false + checkNodeEligibility: true + kataConfigPoolSelector: + matchLabels: + saw.example.com/kata-pool: "true" +``` + +Wait for the selected nodes to complete their machine config updates and +reboots. Check `oc get kataconfig`, `oc get mcp`, and +`oc get runtimeclass kata -o yaml`. The operator creates the `kata` class. SAW +uses a reviewed RuntimeClass YAML with the live handler, scheduling and overhead +fields. Bind that class's name, UID and placement digest in the site YAML below. +The class also needs `openshell.ai/sandbox-attachable=true`. Keep the class and +the owner approval under platform control. + +## Verify a deployed worker + +1. Check the OSC installation and pool. `oc get kataconfig -o yaml` must show + the intended installed nodes and a ready count equal to the selected count. + Check `oc get mcp` and `oc get runtimeclass kata -o yaml`. The expected + handler is `kata`; scheduling must limit Pods to the configured pool. +2. Schedule a restricted nonroot Pod with `runtimeClassName: kata` and a + disposable filesystem RWX claim. Write and read a marker from the guest. + Record the Pod UID, actual node, SCC, and guest boot ID. +3. On that node, match the Pod UID with `crictl inspectp` for the sandbox. + CRI-O can leave the summary runtime handler empty. Its generated OCI + configuration carries the annotation `io.kubernetes.cri-o.RuntimeHandler`. + Correlate that value and the QEMU process with the same sandbox ID. +4. Run the pinned OpenShell workload runtime's `/openshell-sandbox capability-probe` + as a separate Kata Job. Use the workspace's nonroot UID/GID, strict + supplemental groups, `RuntimeDefault` seccomp, all capabilities dropped, + privilege escalation disabled, and the stock unprivileged-port sysctl. + Require exit 0 and `qualified: true`. The executable exercises Landlock, + seccomp notifications, socket policy and DNS. Gateway creation, supervisor + authentication and chat require a separate workspace test. + +## Deployment mode and placement + +The registration API accepts `vm`, `container` and `kata` as deployment modes. +A registration keeps its mode for its entire lifetime. The `container` and +`kata` modes share the Kubernetes backend. For `kata`, the administrator +binds a named class to the owner and site before enrollment. + +The Kubernetes backend accepts administrator bindings for the workload +RuntimeClass name, Kubernetes UID and SHA-256 of the canonical JSON object +containing `handler`, `overhead` and `scheduling`. Missing optional fields hash +as JSON null. The controller checks UID, the stock OpenShell attachable label, +the `kata` handler and the placement digest before attaching resources. Ordinary +container plans retain their original representation. The gateway reads only +the selected class, and the admission policies require that class on both the +Sandbox template and the workload Pod. The supervisor keeps the default +runtime. The portal advertises Kata only when the platform administrator enables the +mode and supplies a reviewed site class and approved profile. + +### Allow time for Kata startup + +OpenShell 0.1.6 uses 300-second startup windows by default. Kata VM startup can +take longer. In Kata mode, the Kubernetes gateway chart sets the provisioning +repair window, runtime bootstrap grace period and sandbox attach request timeout +to 900 seconds. Set `kubernetes.startupTimeouts` to adjust these values. Each +value must be from 1 to 86400 seconds. Container mode keeps the OpenShell +defaults. These longer windows allow more startup time; they do not change the +runtime readiness checks or make an unhealthy sandbox ready. + +## Gated preflight + +Run the read-only preflight before creating a workspace: + +```sh +DEPLOYMENT_MODE=kata OPENSHELL_SAW_NAME=alice \ + SAW_USERS_VALUES_FILE="$PWD/container-users.yaml" \ + make -f Makefile-quickstart saw-preflight +``` + +The command renders the workspace chart and submits it with server-side dry +run. It checks the active API, cluster version, Agent Sandbox APIs, admission +and network inputs, Argo access, storage classes, Vault and External Secrets health, +image digest pins, provider host allowlists, source revisions, KataConfig, +worker placement and RuntimeClass approval. A KataConfig passes only when its +ready node count matches its nonzero node count, installation is not in +progress, and an installed node matches the approved worker selector. This +follows the [OSC status reference](https://docs.redhat.com/en/documentation/openshift_sandboxed_containers/1.12/html/deploying_openshift_sandboxed_containers_on_bare-metal_servers/reference_metal-osc). +The Make path also checks `/dev/kvm` through the existing machine-config +daemon. RHDH and GitOps use KataConfig's installed-node status. The preflight +does not create a namespace, release, registration or credential. Run local +Python tools through `uv`. + +Before RHDH reserves a workspace or writes credentials, it checks the selected +profile's approved admission hashes. If the owner's namespace exists, it reads +the live runtime validation policy and binding. Kata mode also checks the Kata +placement policy and binding. The Make preflight performs the same checks for +an existing workspace namespace. A changed hash stops the request before the +portal writes credentials or the registry entry. Review the rendered policy, +source revision and site approval before changing a hash. + +A first registration has no workspace namespace or policy to compare. In that +case, RHDH, Make, and the static GitOps projector require complete, valid +approval hashes. The projector checks the Kata platform and approved +RuntimeClass before it emits a child Application. If the workspace namespace +exists, it also checks the live admission policies and bindings against the +approved hashes. A failed projector check returns an error to ApplicationSet +and does not return an empty desired set that could prune existing owners. +The projector reads public resources only and cannot read Secrets. + +Argo's enrollment Job verifies a first registration's generated boundary +before it publishes the immutable enrollment binding. That Job runs after +Argo has applied the workspace resources. A first GitOps registration can +therefore create policy resources before a stale policy hash is detected. +Review the pinned policy source and approved hash before that first sync. + +The Make path needs an `oc` context whose API hostname resolves from the +machine that runs Make. RHDH and GitOps run their checks inside the cluster. +RHDH checks the owner and live platform before it writes credentials to Vault. +The `saw-portal-kata-preflight` ClusterRoleBinding must grant read-only access +to both the portal provisioner and the `saw-workspaces-generator` ServiceAccount. +The RHDH create handler runs in the generator Pod. A missing `get` permission +for RuntimeClasses returns `403` and blocks the preflight before enrollment. +The enrollment job checks the approved Kata placement again before it publishes +the immutable runtime binding. A failure stops the request with a fixed reason. +When an API request fails, the enrollment log shows its HTTP status and +Kubernetes API path. It does not show the response body, headers, or query +parameters. Use that status and path to check the matching API permission or +resource. GET requests send an `Accept` header only. PATCH requests also send +the merge-patch `Content-Type`. A `415` on API discovery can point to a wrong +request content type. Other errors keep the current type-only log. +The read-only Make command also needs the Argo-managed installation namespace +to exist so the API server can validate namespaced resources. On a first +`make saw-create`, platform checks run first; the command then creates that +namespace if needed and performs server-side dry run before Helm applies the +workspace chart. + +This gate checks configuration and live API state. It cannot prove that every +node can pull every image or that provider, denied-tool, network-denial, chat, +audit, persistence, restart, stop/start and deletion flows work. Verify those +items on one canary only after the preflight passes. If a canary fails, collect +the pipeline log, Argo sync and health state, controller status, Pod events, +RuntimeClass UID and placement digest before another request. + +If RHDH reports a RuntimeClass preflight error, check that the generator can +read the live class. The fixed reasons separate a missing approval, a missing +class, a name or UID change, a wrong handler, a missing attachable label, an +invalid scheduling rule, and a changed placement digest. Compare the approved +name, UID, handler, scheduling and overhead with the live object. Do not change +an owner approval to match the live class until the platform owner has reviewed +the class placement. + +If RHDH reports `BoundaryApprovalChanged:validation` or a binding reason, the +selected profile's approved hash does not match the live admission resource. +Do not submit another request against that approval. Check the pinned policy +revision and compare the live resource with its rendered source. Update the +site approval only after the platform owner reviews the difference. Existing +registrations keep their immutable approval and need a fresh registration to +use a reviewed change. + +The pinned AIPCC OpenClaw notebook image has `/sandbox` owned by `1000:0` +with mode `0770`. A workspace Pod that uses an OpenShift namespace UID cannot +enter that directory. The shared preflight now returns +`WorkloadImageNotNamespaceAccessible` when an enabled sandbox uses that digest +without a pinned profile image override. This check uses the known image digest; +it is not a general image filesystem inspection. Prepare the notebook image +with `make container-notebook-image BUILD_NS= BUILD_ARCH=amd64` +as described in [Container images](container-images.md#prepare-the-data-science-notebook-for-container-workspaces). +Use the resulting digest in `profileImageOverrides.data-science.notebook` and +grant the scoped `imagePull` access. Review the changed compiled profile and +approve its new digest before a new registration. The validation admission +policy also pins each workload image. Render that policy and approve its new +normalized `validation` digest when an image override changes. Check that the +new policy differs only in the reviewed image reference. The portal generator +reads the site Secret through an environment variable, so restart its +Deployment after a site approval change. Verify the owner Argo Application +and the enrollment bootstrap use the new digest before waiting for the +workspace runtime. A failed registration keeps its old immutable approval +until the normal delete completes, unless it has not yet published an +enrollment binding and the corrected site projection can reconcile it. + +## Standalone workspace on an existing Kata pool + +Start with the [container installation](container-installation.md). Keep its +profile approval, topology overlay, scoped Vault input, RWX claims and owner +registration. The following fields are a fragment to merge into the site file. + +```yaml +users: + - name: alice + values: + deployment: + mode: kata +containerSites: + alice: + kataRuntimeClass: + name: saw-kata + uid: + sha256: +``` + +Manage the class in platform GitOps. The class name used by OpenShell must have +the label `openshell.ai/sandbox-attachable=true`. If you define a separate +class for workspaces, copy the OSC class's handler, overhead and scheduling +constraints when defining it. Inspect the live object before approving it. +Compute the digest using the same canonical serialization as the reconciler. + +```sh +oc get runtimeclass saw-kata -o json | uv run --locked python -c ' +import hashlib, json, sys +obj = json.load(sys.stdin) +placement = {key: obj.get(key) for key in ("handler", "overhead", "scheduling")} +print(obj["metadata"]["uid"]) +print(hashlib.sha256(json.dumps(placement, sort_keys=True, separators=(",", ":")).encode()).hexdigest())' +``` + +Set `DEPLOYMENT_MODE=kata` and run the documented two-stage `make saw-create`. +Add `--kata-runtime-class saw-kata` to each boundary inspection. Review the +Sandbox admission policy and binding along with the gateway configuration and +the Pod policy before enabling enrollment. The same workspace projector and +controller drive both slots through the stock gateway. + +For each slot, inspect the workload Pod's runtime class, UID, SCC, node and +Ready condition. Match its UID to the host-side Kata handler as above. Verify +the associated supervisor Pod uses the default runtime. Finally sign in as the +owner and send a fresh challenge through each slot's browser relay. Record the +observed text independently for each slot. The CUDA slot name identifies a +profile entry; this test exercises chat and does not qualify GPU passthrough. + +### Check a new installation through browser readiness + +After the second `make saw-create` returns successfully, run `make saw-status` +with the same site values and owner. Record its `ready` result, then follow +each slot's owner chat link and verify an independent model reply. For RHDH, +follow the links from the owner's catalog entry. A clean workspace replay uses +a new registration, Vault scope and PVCs. Record separately whether the cluster +already had the shared platform and OSC installed. + +To diagnose a readiness timeout, set `OWNER` and `REGISTRATION` from the site +values. These commands read workload state and component health. + +```sh +oc -n "saw-$OWNER" get sandboxes.agents.x-k8s.io \ + -o 'custom-columns=NAME:.metadata.name,MODE:.spec.operatingMode,READY:.status.conditions[?(@.type=="Ready")].status,REASON:.status.conditions[?(@.type=="Ready")].reason' +oc -n "saw-$OWNER" get pods -o wide +oc -n "saw-ctl-$REGISTRATION" get jobs +oc -n vp-gitops get applications -o json | jq -r --arg prefix "saw-c-$REGISTRATION-relay-" \ + '.items[] | select(.metadata.name | startswith($prefix)) | [.metadata.name, .status.sync.status, .status.health.status] | @tsv' +``` + +The identity bootstrap creates the public gateway CA ConfigMap in +`saw-ctl-`. The enrollment Job and runtime controller also run +there. Separate Argo applications can start these components at different +times. A volume mount can fail before identity bootstrap creates the ConfigMap. +Both components read the named CA through the Kubernetes API and retry if it is +not ready. Grant each ServiceAccount `get` on that ConfigMap name in the +control namespace. Do not grant broad ConfigMap access. If the Job reaches +`BackoffLimitExceeded`, check its Pod events for `FailedMount` and `Evicted` +before retrying. A low ephemeral-storage eviction is a separate cluster +resource issue; record it with the Job failure. + +The runtime controller publishes the immutable enrollment binding only after +the Job completes. If the Job failed, expect no relay binding or Kata worker +Pods. The Argo applications for the gateway and policy can be healthy while the +workspace remains unavailable. + +An enrollment or relay binding Job can succeed before OpenClaw serves the +browser UI. If the stock gateway reports that its supervisor is unavailable +and suspends the workload, inspect the supervisor Pod events and the workload +Pod events in the same namespace. A relay can subsequently exceed its +Deployment progress deadline while waiting for the sandbox's UI port. +Preserve the installer exit status and event timestamps for that run. + +If RHDH waits at `Intent for this registration and update`, check that both +the RHDH workspace generator and portal provisioner can read +`saw-runtime-intent`, `saw-runtime-status`, and `saw-bom-profiles` in the +workspace namespace. A static projector may also need access to enrollment +publications. Keep its ServiceAccount in the same RoleBinding; do not replace +the portal readers with it. The runtime observer Role grants `get` on only +those three ConfigMaps. + +### Verify a replacement + +1. On a disposable registration, record each workload Pod UID and the UID and + bound PV name of each PVC. Write a different marker file under + `/sandbox/workspace` in each slot and record its SHA-256. Sign in as the owner, + request a distinct reply from each slot and keep the value for comparison. +2. For a static registration, commit `desiredState: Stopped` and increment + `intentGeneration`. Wait for the controller to report that generation as + `Stopped` and for both agent Pods to disappear. Commit `desiredState: Running` + with the next generation. Keep the same registration, profile and PVCs. +3. Check that the new generation reaches `GenericReady`, both slots have new + Pod UIDs, and every PVC retains its UID and bound PV. Verify both marker + hashes, the earlier assistant replies in each slot's conversation history, + and a fresh owner chat through each route. Record browser timeouts separately + from successful subsequent attempts. +4. To test an explicit restart while running, increment `intentGeneration` + and set `restartGeneration` to that new value. Repeat the checks and + correlate the new Pod UIDs with the host-side CRI-O sandbox identities. + +## Developer Hub site configuration + +Enable `kata` in `portal.deploymentModes` and configure the owner's reviewed +class in `portal.containerSites..kataRuntimeClass`. Use the same three +fields shown above. The create and create-for-user forms display the profiles +approved for each deployment mode. The selected mode is immutable for the +registration. The server checks the owner's identity, mode, profile and site +placement before reserving credentials or writing to Vault. Owners never +supply a RuntimeClass or a security policy in the form. + +The portal delegates projection and reconciliation to the same components as +Make. A site can enable VM, Container and Kata together. Each active Kubernetes +registration uses the placement approved for that login. Stop, start, model +update, status, chat links and foreground deletion use the existing Kubernetes +lifecycle implementation. diff --git a/docs/provider-driver-qualification.md b/docs/provider-driver-qualification.md new file mode 100644 index 000000000..48225bb74 --- /dev/null +++ b/docs/provider-driver-qualification.md @@ -0,0 +1,174 @@ +# Provider qualification + +The same SAW-BOM, governance profiles and namespace credential proxy apply to +VM and container deployments. The Kubernetes adapter supports a selected +OpenAI, Anthropic or NVIDIA provider; it also supports an explicitly approved +OpenAI-compatible custom endpoint. Anthropic uses `anthropic-messages`, not an +OpenAI compatibility endpoint. No OpenShell driver or agent image change is +required. + +## Inputs + +Use the [container installation](container-installation.md) procedure, including +site approval, digest-pinned images, RHDH enrollment and Vault/ESO provisioning. +For each provider, select it in the approved BOM; do not put a key in the BOM, +intent, application Pod, gateway or controller ledger. + +| Provider type | Native endpoint | Protocol | Placeholder environment variable | +|---|---|---|---| +| `openai` | `https://api.openai.com/v1` | `openai` | `OPENAI_API_KEY` | +| `anthropic` | `https://api.anthropic.com` | `anthropic` | `ANTHROPIC_API_KEY` | +| `nvidia` | `https://integrate.api.nvidia.com/v1` | `openai` | `NVIDIA_API_KEY` | + +Example BOM documents for an Anthropic qualification profile: + +```yaml +apiVersion: saw.redhat.com/v1alpha1 +kind: Providers +spec: + providers: + - name: anthropic + type: anthropic + credentialSecret: inference + credentialSecretKey: api_key +--- +apiVersion: saw.redhat.com/v1alpha1 +kind: Sandboxes +spec: + sandboxes: + - name: notebook + type: openclaw + image: @sha256: + providers: [anthropic] + inference: + providerRef: anthropic + protocol: anthropic + model: claude-sonnet-4-6 + ui: + route: true +``` + +These are the provider/sandbox documents within a normal profile, not new +Kubernetes CRDs. Keep its Workspace document and normal catalog/site approval. +Use equivalent provider/model choices for OpenAI and NVIDIA. The controller +checks the native endpoint against the shipped governance profile and the +enrolled proxy policy. A public model selector is not a credential reference. +Simultaneously registering additional `inference.models` choices is still +rejected by the container compiler; qualify one selected provider per sandbox. + +## Checks + +First run the offline contracts: + +```sh +make test-installer TEST_PATHS=tests/kubernetes_runtime +``` + +Node is required to execute the generated OpenClaw configuration. The Rust +compiler/governance test requires Cargo and cached `serde_json`; CI fetches this +dependency and makes the test mandatory. An offline skip is not a Rust pass. + +For live qualification, use a disposable registration and the ordinary owner +login. Capture the Git revision, component image digests, cluster version, +registration identity, actual gateway driver, workload Pod UID and PVC UIDs. +Verify the stock Kubernetes driver created the workload; running inside a +KubeVirt VM on OpenShift does not qualify the Kubernetes driver. + +With the owner's stock OpenShell CLI configured for that gateway: + +```sh +GATEWAY= +WORKSPACE=default +SANDBOX=notebook +PROVIDER=openai +MODEL=gpt-4.1-mini-2025-04-14 +openshell --gateway "$GATEWAY" sandbox exec -n "$SANDBOX" \ + --workspace "$WORKSPACE" --no-tty -- \ + node --input-type=module -e "$(cat tests/provider-inference-smoke.mjs)" \ + "$PROVIDER" "$MODEL" +``` + +Expected: `httpStatus: 200`, `nonceMatched: true`, exit 0. The script makes one +request from the sandbox, through its normal proxy path, to the provider's fixed +official endpoint. It prints no key, headers, provider error body or model reply. +Repeat with the selected Anthropic and NVIDIA models in their own registered +sandboxes. Do not substitute a direct request from the bastion for this check. +Use `openshell sandbox exec` for this probe: administrative `oc exec` into the +application container does not establish the OpenShell execution session or its +provider placeholder environment, so it does not test the owner's inference path. +Before a model-update test, run the same probe with the proposed model ID and +credential binding. Visibility in the provider's model catalog does not establish +that an inference request will succeed. Record a rejected model separately from +controller convergence and storage persistence. + +Then exercise RHDH/OpenClaw: + +1. **Create:** select **Container** and the approved profile. Wait for the + matching registration/intent to become Ready. Open the owner-only chat link + and request a unique token. Record the selected model and returned token. +2. **Update:** change the public model input through the normal update flow. + Verify a fresh workload, the new model, successful chat and unchanged PVCs. + Inspect the provider/model in the resulting chat transcript as well as the + API probe. A stock agent's automatic configuration migration can change a + model after startup; a correct public selector or direct API reply alone is + insufficient. +3. **Restart:** use the normal stop/start intent. Verify chat and a workspace + canary survive; check a fresh Pod UID and unchanged PVC UIDs. +4. **Isolation:** verify an unknown credential marker and an unapproved + destination are denied, and another owner's browser login cannot open chat. + Keep keys confined to the trusted credential-proxy namespace. +5. **Delete:** delete through RHDH. Verify foreground compute cleanup and the + documented retained workspace/audit data disposition. + +Keep a browser session open across a relay OIDC-token expiry. Check a new chat +connection and reconnection of an existing WebSocket separately, without +automatically resubmitting tool approvals or other non-idempotent operations. +Record any connection gap even if the relay subsequently recovers. + +Record provider-by-provider results in the +[support matrix](provider-driver-support-matrix.md). Configuration tests, +authenticated API inference and browser lifecycle are separate evidence. + +## VM regression alongside containers + +Use a separate disposable **VM** registration with the same approved profile and +provider/model inputs. Repeat both sandboxes' API, owner-browser, shared tool-gate +and credential/owner-denial checks. Resolve the workspace from the stock agent +configuration; do not assume the container path. The tested stock VM images used +`/sandbox/.openclaw/workspace`. + +After the RHDH model update completes, use the documented targeted VM restart: + +```sh +KUBECONFIG= make saw-restart \ + DEPLOYMENT_MODE=vm OPENSHELL_SAW_NAME= \ + SAW_NS= +``` + +Wait for the new VM instance and that instance's installer/BOM completion, not +stale completion markers from before the restart. Require the same VM/root-PVC +UIDs, a new VMI UID, unchanged workspace canaries and original audit records, +and the selected model in fresh API and browser-chat transcripts. Check that +temporary administrator files and trust entries have been removed. + +Delete through RHDH and verify removal of the workspace Applications, namespace, +VM/root PVC, registry entry and registration-scoped Vault keys. Both chat routes +must become unavailable. VM audit lives on the root disk and is deleted with it; +container audit-history retention is a different backend contract. Snapshot +unrelated workspaces before and after to detect unintended Pod/image, VMI or PVC +replacement. A new workspace on an existing SAW platform does not qualify a new +platform installation. + +## Interrupted provider creation + +Before `CreateProvider`, the controller fsyncs a pending provider digest and +the original intent generation to its ledger PVC. The plan contains public +credential-routing markers, never provider keys. If the response is lost, it +reissues the same payload and deterministic request ID under the same installer +principal. The pinned stock gateway replays its durable mutation receipt only +while the resource identity/version still match. Only that receipt authorizes +recording the credential revision; a redacted `GetProvider` is insufficient. + +Changed provider inputs, replaced resources, lost/expired gateway receipts or a +lost local ledger fail closed and require explicit recovery. Do not repair these +by copying a current version into the ledger or disabling ownership checks. diff --git a/docs/provider-driver-support-matrix.md b/docs/provider-driver-support-matrix.md index ac9f53da9..7bd12b242 100644 --- a/docs/provider-driver-support-matrix.md +++ b/docs/provider-driver-support-matrix.md @@ -1,41 +1,174 @@ # SAW TP provider and driver support matrix -**Validation date:** 2026-10-21 +**Updated:** 2026-11-08. Historical VM live results: 2026-10-21. -This matrix records successful inference smoke tests against official provider -APIs. Kubernetes driver integration must be completed before provider validation -can begin. +Keep implementation, offline contracts and live qualification separate. The +Kubernetes integration now compiles native OpenAI, Anthropic and NVIDIA +providers, uses the shared governance/credential policy, and reconciles them +through the stock gateway. This does not by itself qualify a live provider. + +The November 6 OpenAI results below describe the feature revision. The +November 8 security policy was tested separately on OpenShift. It admitted all +five legitimate Pod shapes and rejected the unmarked Pod, forged workload, +unrelated supervisor Secret and workload image replacement. The real Agent +Sandbox controller created the workload and the stock scheduling-gate removal +was admitted. Service/Secret and projector checks admitted the expected shapes +and denied NodePort, trust replacement and service-account-token creation. +The gateway lacked NetworkPolicy creation permission. A fresh static installation served both OpenAI sandbox chats through the +subject-bound relay. The deployed signed tool gate exited before the tool side +effect when an approval could not write its audit record. ## Validation matrix -| Provider | Tested model | Podman driver | Kubernetes driver | -|---|---|---|---| -| OpenAI (primary) | `gpt-6-luna` | Validated | Not tested | -| Anthropic (secondary) | `claude-sonnet-4-6` | Validated | Not tested | -| NVIDIA | `nvidia/nemotron-3.5-lightning-30b-a3b` | Validated | Not tested | +| Provider | Historical VM model | Podman live (2026-10-21) | Kubernetes offline contracts | Kubernetes live qualification | +|---|---|---|---|---| +| OpenAI | `gpt-6-luna` | Historical pass; see scope below | Passed: native protocol/config, marker/host policy, mutable selection | Sandbox API inference, owner chat and audited tool decisions passed on both data-science slots. Standalone model update, stop/start and explicit restart preserved workspace/audit data. RHDH browser create/update/stop/start/delete/recreate passed. Fresh static GitOps and RHDH installations passed inference, owner chat and audited tool decisions on both slots; fresh-connection recovery across relay-token renewal passed; OpenAI VM regression also passed (scope below) | +| Anthropic | `claude-sonnet-4-6` | Historical pass; see scope below | Passed: native Messages protocol/config, marker/host policy, mutable selection | Not qualified | +| NVIDIA | `nvidia/nemotron-3.5-lightning-30b-a3b` | Historical pass; see scope below | Passed: OpenAI protocol/config, marker/host policy, slash-qualified model selection | Not qualified | + +Offline cases use synthetic credentials and Node execution of the actual +compiled OpenClaw configuration. Provider recovery tests cover a lost create +response, process restart and a later intent generation; drift, replacement, +changed markers and missing receipts fail closed. The Rust admission contract +test consumes the actual Helm/compiler harness output (3 volumes/5 mounts, +also with 16 extra data volumes) and rejects altered aliases. CI requires that +Rust test to execute, rather than silently treating a missing toolchain as a pass. -## Test results +## Historical VM results -All three providers passed authenticated inference requests from inside the -sandbox through the in-VM OpenShell sandbox proxy, before namespace credential isolation. The namespace-proxy path is **not yet re-validated** for this matrix. Each API returned **HTTP 200** -with a response matching the test prompt. +The 2026-10-21 results covered authenticated inference from a sandbox through +the in-VM OpenShell sandbox proxy, before namespace credential isolation. Each +API returned **HTTP 200** with a response matching the test prompt. These results +do not qualify the current namespace-proxy path or candidate installation. OpenClaw chat also returned responses from all three providers. NVIDIA passed with both an explicit model selection and the configured notebook default. -**Environment:** OpenShift with Podman running inside KubeVirt VMs; -`default/notebook` in `saw-provider-test` (OpenAI and Anthropic) and -`saw-nvidia-test` (NVIDIA). Running on OpenShift does not validate the -OpenShell Kubernetes driver. +The historical backend was Podman inside KubeVirt VMs on OpenShift. This does +not validate the OpenShell Kubernetes driver. Environment identities and raw +live evidence belong in the separate qualification record, not this matrix. + +## Candidate VM regression + +A fresh RHDH VM registration passed OpenAI API inference and owner browser chat +on both data-science sandboxes, with `gpt-4.1-2025-04-14` confirmed in response +transcripts. Shared tool policy passed allowed reads, policy denial, owner +rejection and owner approval; actual file effects and six append-only audit +records per sandbox were verified. Anonymous/foreign-owner access and credential +marker/destination denials passed. Temporary installer administrator credentials +were removed and the gateway trust bundle restored. RHDH model update followed +by the documented targeted VM restart passed +`gpt-4.1-mini-2025-04-14` inference on both sandboxes, with the same VM/root PVC, +a fresh VM instance, and unchanged original workspace files and audit hashes. +Both owner browser chats also passed after the restart. RHDH deletion removed +the four workspace Applications, VM namespace/root PVC, registry entry and +registration-scoped Vault keys; both retired chat routes became unavailable. +Unlike container audit history, VM audit records live on the root disk and are +deleted with it. Pre-existing workspaces retained their Pod/image, VM-instance +and PVC identities. This is a workspace regression on an installed platform, +not a fresh platform-install qualification. ## Remaining work -Complete Kubernetes compute driver integration, then validate OpenAI, Anthropic, -and NVIDIA using the same inference checks and update this matrix. +Use [provider qualification](provider-driver-qualification.md) to reproduce the +completed OpenAI checks and qualify additional providers/profiles. Keep immutable +image/Git revisions and actual workload identities in the separate evidence +record; the completed OpenAI lifecycle is not outstanding work. +Anthropic and NVIDIA require separate native-provider runs with real credentials +before their live cells can pass. Do not infer their results from OpenAI success. + +The current OpenAI result uses the official remote API through the Kubernetes +credential proxy. The CUDA-named sandbox does not imply local GPU inference. +The stock gateway also passed committed-provider-response recovery with a fresh +temporary administrator identity; the same provider ID/version was recovered +and the disposable provider and temporary credentials were removed afterward. +These checks do not replace RHDH lifecycle or final clean-install qualification. + +Current qualification also exposed failures that remain distinct from the +initial inference pass: + +- An earlier model update reached Ready and preserved storage, but stock + OpenClaw restored/migrated the notebook's model configuration. The integration + now publishes managed configuration through the native batch command and uses + an explicit managed provider ID. Fresh standalone and RHDH sandboxes passed + actual API inference and owner chat with matching requested/response models; + both RHDH browser UIs also passed. A standalone model update then passed actual + API inference and owner chat on both replacement application Pods, with the + selected model reflected in provider response transcripts. Workspace canaries, + tool effects and original audit records survived unchanged. A subsequent + standalone stop/start and explicit restart passed the same persistence, + inference and chat checks. The post-restart controller observation found no + remaining temporary administrator sessions or private credential directories. + RHDH browser model update also passed with fresh application Pods, unchanged + workspace PVCs and the selected model confirmed by both provider transcripts + and a new browser-chat response. RHDH browser stop passed with no application + Pods, unavailable chat routes and retained workspace/audit claims. RHDH browser + start then passed actual inference and owner chat on both fresh application + Pods, with the original workspace files, tool effects and audit records intact. + Standalone deletion retained its two audit-history claims and left other + workspace namespaces unchanged. RHDH browser deletion removed the runtime + and registration while retaining both audit-history claims. Recreating the + workspace through RHDH produced a fresh registration and passed actual API + inference, owner browser chat and six expected audit events on each slot. +- A RHDH tool run reached its decisions, but the stock CLI closed an exec relay + before reporting the verification command's exit status. Independent read-only + checks confirmed that slot's expected file effects and audit records. The + combined test remains failed. A separate test with new fixtures after the + RHDH model update passed both slots, including stock-exec verification and six + expected audit records per slot. This does not resolve the earlier + intermittent transport failure or justify retrying mutating commands. +- The stock CLI's browser-forward process briefly closes its listener when + its OIDC token expires. Both relays passed new HTTP requests and fresh + authenticated WebSocket handshakes initiated within measured natural renewal + gaps, without request replay. This qualifies sampled new-connection recovery; + survival or automatic reconnection of an already-open WebSocket is not + established, and uninterrupted chat is not guaranteed. +- Initial deployment of the shared transport exposed a Python import collision + with the adjacent OIDC helper. Isolated mode fixes that startup failure, with + a subprocess regression reproducing the old failure. GitOps recovery and + subsequent inference/chat passed. Separate fresh static GitOps and RHDH + installations of the final integration revision then passed actual inference, + owner chat and audited tool decisions on both data-science slots. They needed + no manual workload repair or operator retry. The static identity bootstrap + and one relay-binding Job per installation required automatic retries; these + are not zero-retry installation results. +- VM regression exposed parent Application sync oscillation: Argo adds + PreDelete hook-state finalizers and the parent repeatedly removed them. Both + portal and static projector parents now preserve only those two exact entries + during comparison and sync. Automatic GitOps convergence passed without + changing workload revisions, and existing workspace identities were unchanged. +- An identity-service upgrade rejected the existing service-account admin API + tokens and blocked RHDH enrollment. The shared bootstrap now assigns the + supported built-in Admin API scope; live reads recovered for all three + service clients with unchanged roles. Bootstrap runs during Sync to avoid + waiting for the identity-dependent ApplicationSet's health. The portal also + preserves the prior workspace projection on lookup failure. The subsequent + fresh OpenAI registrations above passed; + that does not qualify other providers or a fresh platform installation. + +Credential-marker and destination denials passed in isolated live checks. +Separate direct-connect probes from both application Pods timed out to two +currently resolved OpenAI IPv4 addresses on TCP/443, while mediated inference +succeeded before and after; the stock deny-all network policies remained in +place. This bounds the network result to the tested destinations and transport. +An authenticated second owner was denied access to the first owner's notebook +in the browser, with the proxy confirming the rejected identity. Broader +profile/provider coverage remains unqualified; the completed OpenAI lifecycle +and final fresh-workspace replays are described above. +Keep detailed failures and immutable revision evidence in the separate +qualification record. + +The container compiler currently selects one inference provider per sandbox; +additional simultaneous `inference.models` chat choices remain unsupported and +fail closed. Kata/confidential runtime qualification is outside this matrix. ## Scope -Validation covers inference in existing workspaces after credential and model -updates. Clean deployment, restart recovery, tools, and memory embeddings are -not validated. A memory-embedding credential error was observed separately -while chat succeeded. +The historical live scope was inference in existing workspaces after credential +and model updates. It did not qualify clean installation, restart recovery, +tools, or memory embeddings. The current container evidence includes +installation, browser lifecycle, persistence and denial checks described above. +Candidate VM regression passed within the OpenAI/data-science scope. A fresh +platform installation, established-WebSocket continuity and other providers +remain unqualified. Offline tests and image builds do not satisfy those live +criteria. diff --git a/docs/rhdh-user-guide.md b/docs/rhdh-user-guide.md index be0fa8d5d..7591d8e04 100644 --- a/docs/rhdh-user-guide.md +++ b/docs/rhdh-user-guide.md @@ -166,14 +166,25 @@ oc logs -n saw-portal deploy/saw-workspaces-generator --tail=20 ## 4. Request a workspace 1. In RHDH: **Create** → **Create or update my agent workspace** → **Choose**. -2. **Profile**: pick one, e.g. `data-science`. The form now asks only for - that profile's keys: +2. Select the deployment and profile. **Virtual machine** is the default. + **Container** appears when the site advertises an approved container profile. + The portal also checks that the signed-in owner is approved for the selected + profile. For the GitOps inputs, digest check and first enrollment, follow + [first-owner container approval](container-installation.md#approve-the-first-rhdh-container-profile). +3. Enter credentials for the provider configured in the selected profile. + For the shipped NVIDIA `data-science` profile these are: - `inference: API key`: the NVIDIA API key (build.nvidia.com); - `web-search: API key`: the Brave Search API key. "Sandbox web UIs" lists the sandboxes that get their own route (`notebook (default)`). -3. **Review** → **Create**. +4. **Review** → **Create**. + +For container registrations the first two tasks, `register` and +`argo-cd-apps`, verify the registration and generated Applications. +The run can still be waiting for site enrollment, storage or the sandboxes. +Wait for readiness and send a message through the owner's chat link to verify +the deployed workspace. Argo CD builds the workspace as before; pipeline `saw-workspace-create` follows it, one task per stage, and the run page shows one step per task diff --git a/docs/self-service-portal.md b/docs/self-service-portal.md index cd8112cb8..f70c1d5b9 100644 --- a/docs/self-service-portal.md +++ b/docs/self-service-portal.md @@ -1,11 +1,29 @@ # Self-service agent workspaces (RHDH) Users create their own Secure Agent Workspace from Red Hat Developer Hub -(RHDH). They pick a SAW-BOM profile, enter the keys that profile needs, and -get namespace `saw-` with a VM, the profile's OpenShell sandboxes, and +(RHDH). They pick an enabled deployment backend and a SAW-BOM profile, enter the keys that profile needs, and +get namespace `saw-` with the profile's OpenShell sandboxes and routes to the sandbox web UIs. Nothing goes into Git: the keys go to Vault, and the workspace is an entry in an in-cluster registry that Argo CD builds. +VM remains the default. Container and Kata modes use the same forms, +registration, provider inputs and lifecycle pipelines. Container mode uses a +Kubernetes runtime adapter. Kata mode applies an approved RuntimeClass to the +OpenShell workers. An operator enables the deployment mode and supplies an +approved owner site. Backend selection is recorded in the registration and +cannot silently migrate an existing workspace. + +Owner sites can come from chart values or from a Kubernetes Secret. Use +`portal.containerSitesSecretName` when the pattern repository is public or the +site includes owner or cluster identifiers. The Secret must be in +`portal.namespace` and contain the JSON map under the `container-sites.json` +key. Git stores only the Secret name. With this source, the form shows catalog +profiles for enabled modes; the create pipeline still checks the signed-in +owner, RuntimeClass and approved profile against the Secret. +See [container installation](container-installation.md#pattern-and-rhdh-enablement) +and [architecture](container-workspaces.md). The VM-specific flow below describes +the default backend; container component ownership and data paths are in those guides. + The step-by-step walkthrough and manual test is in [rhdh-user-guide.md](rhdh-user-guide.md). This replaces the Helm-install flow of PR #33 (RHDH portal, by @pmatouse) with @@ -239,6 +257,14 @@ provisioner, with the single parameter `request=saw-req-*`. A request is deleted when it has been handled, refused when older than an hour, and removed by the CronJob `saw-portal-cleanup` if no pipeline ever handled it. +The pipeline and status service fetch RHDH signing keys from its configured +HTTPS route using the mounted CA bundle. Redirects are rejected. The status +service caches each of RHDH's two JWKS documents for five minutes. +On an unfamiliar signing key ID it refetches the corresponding configured +endpoint, with at most one additional fetch per endpoint every five seconds. +A matching, valid ES256 signature is still required. An unpublished key or a +bad signature receives HTTP 401. + A workspace is refused when its namespace exists without the portal label (the user is in `overrides/saw-users.yaml`), when one of the Argo CD applications it needs belongs to someone else, or when the user name ends in @@ -472,10 +498,29 @@ Before the first sync: `oc get argocd -n vp-gitops -o jsonpath='{.items[0].spec.applicationSet}'` must not be empty. -A PostSync Job creates or updates the Keycloak client `rhdh` (confidential, +A Sync-hook Job creates or updates the Keycloak client `rhdh` (confidential, redirect `https:///api/auth/oidc/handler/frame`) with the secret from Vault, also in a realm that existed before. +Keycloak 26.6.8+ requires `kc-admin-api` scope and the realm Admin REST URL +in administrative access-token audiences. The shared bootstrap helper assigns +that built-in scope as a default to the `rhdh` catalog/portal client, VM owner +resolver and redirect registrar. Their existing realm-management roles stay +unchanged; browser relay clients keep only their OpenShell audience. Earlier +Keycloak versions without the built-in scope are left unchanged. API failures +stop bootstrap instead of bypassing audience verification. + +The RHDH Sync-hook Job performs this migration; changes to the shared helper +also roll the resolver and registrar through their normal bootstrap init +containers. No credentials, token-request changes or custom Keycloak image +are required. When upgrading a previously installed Keycloak, include these +bootstrap reconciliations in the platform rollout; a server-only upgrade does +not rerun existing init containers. The Job shares the ordinary sync wave with +its Secret, RBAC and ConfigMaps; it can repair identity access while dependent +controllers are unhealthy. Waiting for PostSync would create a health dependency +cycle. See the +[Keycloak 26.6.8 migration requirement](https://github.com/keycloak/keycloak/blob/26.6.8/docs/documentation/upgrading/topics/changes/changes-26_6_8.adoc#admin-rest-api-now-enforces-an-audience-check). + RHDH is at `https://backstage-developer-hub-rhdh.apps.`. Sign in with a realm user; user names must be lowercase DNS labels of at most 19 characters (they name the VM). diff --git a/docs/testing.md b/docs/testing.md index 9e77ddc8e..2198121b1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -14,7 +14,7 @@ Update both dependency declarations and run `uv lock` when you add a package. For the complete suite, use Ubuntu 24.04 with Python 3.12 or 3.14, GNU Make, Bash, Helm 3.17.3, Node 22, Go from `credential-proxy/go.mod`, OpenSSL, Git, OpenSSH tools, `jq`, `envsubst` (gettext), GNU coreutils (`shred`, `timeout`, -`stat`, `install`), GNU tar, `flock`, and `pgrep`/`pkill`. Some unit checks also +`stat`, `install`), GNU tar, `find`, `flock`, and `pgrep`/`pkill`. Some unit checks also use `systemd-analyze`. The Go binary check can download Go modules on its first run. Embedded Rust checks need Cargo and cached `serde_json` dependencies. They diff --git a/image-builder-charts/helm/governance-interceptor-image/files/Dockerfile b/image-builder-charts/helm/governance-interceptor-image/files/Dockerfile index 023ecf209..aba29a8ce 100644 --- a/image-builder-charts/helm/governance-interceptor-image/files/Dockerfile +++ b/image-builder-charts/helm/governance-interceptor-image/files/Dockerfile @@ -16,194 +16,13 @@ RUN git init . \ && git checkout --detach FETCH_HEAD WORKDIR /build/${CONTEXT_DIR} -# Workaround for NVIDIA/OpenShell#3929 (verified against v0.1.6): the gateway hashes provider -# profile annotations in map order, so a profile with more than one annotation -# gives a different provider_env_revision on every call and no sandbox with a -# provider becomes ready ("Startup configuration did not stabilize"). Keep only -# the signature JWT, which already carries the hash (claim) and key id (header); -# the gateway treats the annotations as opaque. Drop this once #3929 is fixed. -RUN grep -q 'insert(PROFILE_HASH_ANNOTATION' src/main.rs \ - && grep -q 'PROFILE_SIGNATURE_KID_ANNOTATION.to_string()' src/main.rs \ - && perl -0pi -e 's/\n profile\n \.annotations\n \.insert\(PROFILE_HASH_ANNOTATION\.to_string\(\), profile_hash\);\n profile\.annotations\.insert\(\n PROFILE_SIGNATURE_KID_ANNOTATION\.to_string\(\),\n policy_signer\.kid\(\)\.to_string\(\),\n \);\n/\n/' src/main.rs \ - && ! grep -q 'insert(PROFILE_HASH_ANNOTATION' src/main.rs -# Agent driver-config guard. Every agent needs read-only gate and append-only audit -# mounts; an OpenClaw harness adds a third read-only mount. Only the installer -# mTLS admin can request these exact combinations. Templates are refused before -# resolution because their effective driver config cannot be inspected here. -RUN grep -q 'Ok(validate_create_sandbox(' src/main.rs \ - && perl -0pi -e 's# \("CreateSandbox", interceptor_evaluation::Phase::Validate\(_\)\) => \{\n Ok\(validate_create_sandbox\(# ("CreateSandbox", interceptor_evaluation::Phase::Validate(_)) => {\n let gate = validate_sandbox_driver_config(&operation, &evaluation.principal);\n if !gate.allowed {\n return Ok(gate);\n }\n Ok(validate_create_sandbox(#' src/main.rs \ - && grep -q 'let gate = validate_sandbox_driver_config(' src/main.rs -RUN printf '%s\n' \ -'' \ -'const SAW_ADMIN_ROLE: &str = "openshell-admin";' \ -'fn validate_sandbox_driver_config(operation: &Value, principal: &HashMap) -> InterceptorResult {' \ -' if operation.get("workloadTemplate").or_else(|| operation.get("workload_template"))' \ -' .and_then(Value::as_str).is_some_and(|name| !name.is_empty()) {' \ -' return deny("workload templates are not allowed: driver config cannot be verified before template resolution");' \ -' }' \ -' validate_driver_config(operation.pointer("/spec/template/driverConfig")' \ -' .or_else(|| operation.pointer("/spec/template/driver_config")), principal)' \ -'}' \ -'' \ -'fn valid_volume(mount: &Value, prefix: &str, target: &str, read_only: bool) -> bool {' \ -' let Some(fields) = mount.as_object() else { return false; };' \ -' fields.len() == 4' \ -' && fields.get("type").and_then(Value::as_str) == Some("volume")' \ -' && fields.get("source").and_then(Value::as_str)' \ -' .is_some_and(|source| source.starts_with(prefix) && source.len() > prefix.len())' \ -' && fields.get("target").and_then(Value::as_str) == Some(target)' \ -' && fields.get("read_only") == Some(&Value::Bool(read_only))' \ -'}' \ -'' \ -'fn validate_driver_config(' \ -' driver_config: Option<&Value>,' \ -' principal: &HashMap,' \ -') -> InterceptorResult {' \ -' let Some(config) = driver_config else { return allow(); };' \ -' let Some(drivers) = config.as_object() else {' \ -' return deny("driver config must be an object");' \ -' };' \ -' if drivers.is_empty() { return allow(); }' \ -' let is_admin = principal.get("kind").map(String::as_str) == Some("user")' \ -' && principal.get("provider").map(String::as_str) == Some("mtls")' \ -' && principal.get("roles").is_some_and(|roles| {' \ -' roles.split(",").any(|role| role.trim() == SAW_ADMIN_ROLE)' \ -' });' \ -' if !is_admin {' \ -' return deny("driver config may only be submitted by a platform admin mTLS identity");' \ -' }' \ -' let Some(podman) = drivers.get("podman").and_then(Value::as_object) else {' \ -' return deny("driver config must contain only podman agent mounts");' \ -' };' \ -' if drivers.len() != 1 || podman.len() != 1 {' \ -' return deny("driver config contains unsupported drivers or options");' \ -' }' \ -' let Some(mounts) = podman.get("mounts").and_then(Value::as_array) else {' \ -' return deny("driver config mounts must be an array");' \ -' };' \ -' let valid = match mounts.as_slice() {' \ -' [gate, audit] => valid_volume(gate, "saw-gate-", "/sandbox/.saw/trusted", true)' \ -' && valid_volume(audit, "saw-audit-", "/sandbox/.saw/audit", false),' \ -' [harness, gate, audit] => valid_volume(harness, "saw-harness-", "/sandbox/harness", true)' \ -' && valid_volume(gate, "saw-gate-", "/sandbox/.saw/trusted", true)' \ -' && valid_volume(audit, "saw-audit-", "/sandbox/.saw/audit", false),' \ -' _ => false,' \ -' };' \ -' if !valid {' \ -' return deny("driver config must contain gate and audit volumes, with an optional read-only harness");' \ -' }' \ -' allow()' \ -'}' \ -'' \ -'#[cfg(test)]' \ -'mod agent_driver_tests {' \ -' use super::*;' \ -' use serde_json::json;' \ -'' \ -' fn admin() -> HashMap {' \ -' HashMap::from([("kind".into(), "user".into()), ("provider".into(), "mtls".into()),' \ -' ("roles".into(), "viewer, openshell-admin".into())])' \ -' }' \ -' fn gate_and_audit() -> Value {' \ -' json!({"podman":{"mounts":[' \ -' {"type":"volume", "source":"saw-gate-default-notebook-12345678",' \ -' "target":"/sandbox/.saw/trusted", "read_only":true},' \ -' {"type":"volume", "source":"saw-audit-default-notebook-12345678",' \ -' "target":"/sandbox/.saw/audit", "read_only":false}' \ -' ]}})' \ -' }' \ -' fn with_harness() -> Value {' \ -' let mut config = gate_and_audit();' \ -' config["podman"]["mounts"].as_array_mut().unwrap().insert(0,' \ -' json!({"type":"volume", "source":"saw-harness-default-notebook-12345678",' \ -' "target":"/sandbox/harness", "read_only":true}));' \ -' config' \ -' }' \ -' #[test]' \ -' fn rejects_unresolved_workload_templates() {' \ -' for key in ["workloadTemplate", "workload_template"] {' \ -' let mut operation = json!({});' \ -' operation[key] = json!("stored-template");' \ -' assert!(!validate_sandbox_driver_config(&operation, &admin()).allowed);' \ -' operation[key] = json!("");' \ -' assert!(validate_sandbox_driver_config(&operation, &HashMap::new()).allowed);' \ -' }' \ -' for key in ["driverConfig", "driver_config"] {' \ -' let mut operation = json!({"spec":{"template":{}}});' \ -' operation["spec"]["template"][key] = with_harness();' \ -' assert!(validate_sandbox_driver_config(&operation, &admin()).allowed);' \ -' assert!(!validate_sandbox_driver_config(&operation, &HashMap::new()).allowed);' \ -' }' \ -' }' \ -' #[test]' \ -' fn accepts_exact_installer_combinations_and_empty_config() {' \ -' assert!(validate_driver_config(Some(&gate_and_audit()), &admin()).allowed);' \ -' assert!(validate_driver_config(Some(&with_harness()), &admin()).allowed);' \ -' assert!(validate_driver_config(None, &HashMap::new()).allowed);' \ -' assert!(validate_driver_config(Some(&json!({})), &HashMap::new()).allowed);' \ -' }' \ -' #[test]' \ -' fn rejects_oidc_admin_and_nonadmin_mtls() {' \ -' let mut principal = admin();' \ -' principal.insert("provider".into(), "oidc".into());' \ -' assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed);' \ -' principal = admin();' \ -' principal.insert("roles".into(), "viewer".into());' \ -' assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed);' \ -' principal = admin();' \ -' principal.insert("kind".into(), "service".into());' \ -' assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed);' \ -' }' \ -' #[test]' \ -' fn rejects_wrong_source_target_type_and_mode() {' \ -' for index in 0..2 {' \ -' for (key, value) in [("type", json!("bind")), ("source", json!("other")),' \ -' ("source", json!("saw-gate-")), ("target", json!("/other")),' \ -' ("read_only", json!("wrong"))] {' \ -' let mut config = gate_and_audit();' \ -' config["podman"]["mounts"][index][key] = value;' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed, "accepted {config}");' \ -' }' \ -' }' \ -' let mut config = gate_and_audit();' \ -' config["podman"]["mounts"][0]["read_only"] = json!(false);' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed);' \ -' config = gate_and_audit();' \ -' config["podman"]["mounts"][1]["read_only"] = json!(true);' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed);' \ -' }' \ -' #[test]' \ -' fn rejects_other_drivers_options_and_mount_combinations() {' \ -' for config in [json!({"docker":{"mounts":[]}}), json!({"podman":{"mounts":[]}}),' \ -' json!({"podman":{}}), json!({"podman":{"privileged":true}})] {' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed, "accepted {config}");' \ -' }' \ -' let mut config = gate_and_audit();' \ -' config["podman"]["privileged"] = json!(true);' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed);' \ -' config = gate_and_audit();' \ -' config["podman"]["mounts"][0]["options"] = json!(["rw"]);' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed);' \ -' config = gate_and_audit();' \ -' config["podman"]["mounts"].as_array_mut().unwrap().swap(0, 1);' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed);' \ -' config = gate_and_audit();' \ -' config["podman"]["mounts"].as_array_mut().unwrap().pop();' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed);' \ -' config = with_harness();' \ -' config["podman"]["mounts"].as_array_mut().unwrap().push(json!(null));' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed);' \ -' }' \ -' #[test]' \ -' fn rejects_malformed_envelopes() {' \ -' for config in [json!(null), json!(true), json!("bad"), json!([]),' \ -' json!({"podman":null}), json!({"podman":{"mounts":null}}),' \ -' json!({"podman":{"mounts":[null]}})] {' \ -' assert!(!validate_driver_config(Some(&config), &admin()).allowed, "accepted {config}");' \ -' }' \ -' }' \ -'}' \ ->> src/main.rs +# The SAW extension is ordinary Rust source. The reviewed patch connects it +# through the governance API and retains the annotation-order workaround for NVIDIA/OpenShell#3929. +# A different upstream source must satisfy the patch exactly before compilation. +COPY saw-extension/driver_config.rs src/saw_driver_config.rs +COPY saw-extension/integration.patch /tmp/saw-integration.patch +RUN git -C /build apply --check --directory="${CONTEXT_DIR}" /tmp/saw-integration.patch \ + && git -C /build apply --directory="${CONTEXT_DIR}" /tmp/saw-integration.patch RUN cargo test --locked --release --target-dir /out agent_driver_tests \ && cargo build --locked --release --target-dir /out diff --git a/image-builder-charts/helm/governance-interceptor-image/files/saw-extension/driver_config.rs b/image-builder-charts/helm/governance-interceptor-image/files/saw-extension/driver_config.rs new file mode 100644 index 000000000..6b613cbfe --- /dev/null +++ b/image-builder-charts/helm/governance-interceptor-image/files/saw-extension/driver_config.rs @@ -0,0 +1,346 @@ +// SAW admission extension. Called through the stock governance interceptor API. +use super::*; + +const SAW_ADMIN_ROLE: &str = "openshell-admin"; +pub(crate) fn validate_sandbox_driver_config(operation: &Value, principal: &HashMap) -> InterceptorResult { + if operation.get("workloadTemplate").or_else(|| operation.get("workload_template")) + .and_then(Value::as_str).is_some_and(|name| !name.is_empty()) { + return deny("workload templates are not allowed: driver config cannot be verified before template resolution"); + } + validate_driver_config(operation.pointer("/spec/template/driverConfig") + .or_else(|| operation.pointer("/spec/template/driver_config")), principal) +} + +fn valid_volume(mount: &Value, prefix: &str, target: &str, read_only: bool) -> bool { + let Some(fields) = mount.as_object() else { return false; }; + fields.len() == 4 + && fields.get("type").and_then(Value::as_str) == Some("volume") + && fields.get("source").and_then(Value::as_str) + .is_some_and(|source| source.starts_with(prefix) && source.len() > prefix.len()) + && fields.get("target").and_then(Value::as_str) == Some(target) + && fields.get("read_only") == Some(&Value::Bool(read_only)) +} + +fn validate_driver_config( + driver_config: Option<&Value>, + principal: &HashMap, +) -> InterceptorResult { + let Some(config) = driver_config else { return allow(); }; + let Some(drivers) = config.as_object() else { + return deny("driver config must be an object"); + }; + if drivers.is_empty() { return allow(); } + let provider = principal.get("provider").map(String::as_str); + let is_operator = principal.get("kind").map(String::as_str) == Some("operator") + && provider == Some("mtls"); + let is_admin_user = principal.get("kind").map(String::as_str) == Some("user") + && provider == Some("mtls") + && principal.get("roles").is_some_and(|roles| { + roles.split(",").any(|role| role.trim() == SAW_ADMIN_ROLE) + }); + let is_admin = is_operator || is_admin_user; + if !is_admin { + return deny("driver config may only be submitted by a platform admin mTLS identity"); + } + if let Some(kubernetes) = drivers.get("kubernetes") { + return if drivers.len() == 1 && valid_kubernetes_driver(kubernetes) { allow() } + else { deny("Kubernetes driver config must use the approved retained-PVC envelope") }; + } + let Some(podman) = drivers.get("podman").and_then(Value::as_object) else { + return deny("driver config must contain only podman agent mounts"); + }; + if drivers.len() != 1 || podman.len() != 1 { + return deny("driver config contains unsupported drivers or options"); + } + let Some(mounts) = podman.get("mounts").and_then(Value::as_array) else { + return deny("driver config mounts must be an array"); + }; + let valid = match mounts.as_slice() { + [gate, audit] => valid_volume(gate, "saw-gate-", "/sandbox/.saw/trusted", true) + && valid_volume(audit, "saw-audit-", "/sandbox/.saw/audit", false), + [harness, gate, audit] => valid_volume(harness, "saw-harness-", "/sandbox/harness", true) + && valid_volume(gate, "saw-gate-", "/sandbox/.saw/trusted", true) + && valid_volume(audit, "saw-audit-", "/sandbox/.saw/audit", false), + _ => false, + }; + if !valid { + return deny("driver config must contain gate and audit volumes, with an optional read-only harness"); + } + allow() +} + +// SAW integration policy: only a platform mTLS admin may select retained PVCs. +// Live namespace/PVC UID and attachment approval stay in the reconciler/driver. +fn k8s_name(value: &str) -> bool { + !value.is_empty() && value.len() <= 63 + && value.as_bytes()[0].is_ascii_alphanumeric() + && value.as_bytes()[value.len()-1].is_ascii_alphanumeric() + && value.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == 45) +} +fn k8s_quantity(value: &Value) -> bool { + let Some(text) = value.as_str() else { return false; }; + let digits = text.bytes().take_while(|b| b.is_ascii_digit()).count(); + digits > 0 && text.as_bytes()[0] != 48 + && ["", "m", "Ki", "Mi", "Gi", "Ti"].contains(&&text[digits..]) +} +fn valid_kubernetes_driver(config: &Value) -> bool { + let Some(driver) = config.as_object() else { return false; }; + if driver.keys().any(|k| !["volumes", "containers"].contains(&k.as_str())) { return false; } + let Some(volumes) = driver.get("volumes").and_then(Value::as_array) else { return false; }; + let Some(containers) = driver.get("containers").and_then(Value::as_object) else { return false; }; + let Some(agent) = containers.get("agent").and_then(Value::as_object) else { return false; }; + if containers.len() != 1 || agent.len() != 2 { return false; } + let Some(mounts) = agent.get("volume_mounts").and_then(Value::as_array) else { return false; }; + let Some(resources) = agent.get("resources").and_then(Value::as_object) else { return false; }; + for (group, values) in resources { + if !["requests", "limits"].contains(&group.as_str()) { return false; } + let Some(values) = values.as_object() else { return false; }; + if values.iter().any(|(k,v)| !["cpu", "memory", "ephemeral-storage"].contains(&k.as_str()) || !k8s_quantity(v)) { return false; } + } + let gate = mounts.get(1).and_then(|m| m.get("name")).and_then(Value::as_str) == Some("saw-gate"); + let reserved = if gate { 3 } else { 1 }; + if volumes.len() < reserved || volumes.len() > reserved + 16 || mounts.len() < volumes.len() { return false; } + // The briefing harness reuses two approved PVCs. These are the only aliases + // allowed; base mounts still have a one-to-one ordered volume mapping. + let aliases = &mounts[volumes.len()..]; + if !aliases.is_empty() { + if !gate || aliases.len() != 2 { return false; } + for (mount, name, path, ro) in [ + (&aliases[0], "saw-gate", "/sandbox/harness", true), + (&aliases[1], "saw-workspace", "/sandbox/briefing", false), + ] { + let Some(fields) = mount.as_object() else { return false; }; + if fields.len() != 4 || fields.get("name").and_then(Value::as_str) != Some(name) + || fields.get("mount_path").and_then(Value::as_str) != Some(path) + || fields.get("read_only").and_then(Value::as_bool) != Some(ro) { return false; } + let Some(sub) = fields.get("sub_path").and_then(Value::as_str) else { return false; }; + if ro { + let Some(digest) = sub.strip_prefix("harness-") else { return false; }; + if digest.len() != 64 || !digest.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { return false; } + } else if sub != "briefing" { return false; } + } + } + let mut paths: Vec = Vec::new(); + let mut claims: Vec = Vec::new(); + for (index, (volume, mount)) in volumes.iter().zip(mounts.iter()).enumerate() { + let Some(v) = volume.as_object() else { return false; }; + let Some(m) = mount.as_object() else { return false; }; + let name = match (index, gate) { + (0, _) => "saw-workspace".into(), (1, true) => "saw-gate".into(), (2, true) => "saw-audit".into(), + _ => format!("data-{}", index-reserved), + }; + let reserved_mount = gate && (index == 1 || index == 2); + if v.len() != 2 || m.len() != (if reserved_mount { 4 } else { 3 }) || v.get("name").and_then(Value::as_str) != Some(name.as_str()) + || m.get("name").and_then(Value::as_str) != Some(name.as_str()) { return false; } + let Some(pvc) = v.get("persistent_volume_claim").and_then(Value::as_object) else { return false; }; + if pvc.len() != 2 { return false; } + let Some(claim) = pvc.get("claim_name").and_then(Value::as_str) else { return false; }; + let Some(read_only) = pvc.get("read_only").and_then(Value::as_bool) else { return false; }; + if (!reserved_mount && (claim.starts_with("saw-gate-") || claim.starts_with("saw-audit-"))) + || !k8s_name(claim) || claims.iter().any(|c| c == claim) + || m.get("read_only").and_then(Value::as_bool) != Some(read_only) { return false; } + let Some(path) = m.get("mount_path").and_then(Value::as_str) else { return false; }; + if index == 0 { + if path != "/sandbox/workspace" || read_only { return false; } + } else if gate && index == 1 { + if m.get("sub_path").and_then(Value::as_str) != Some("bundle") || path != "/sandbox/.saw/trusted" || !read_only || !claim.starts_with("saw-gate-") { return false; } + } else if gate && index == 2 { + if m.get("sub_path").and_then(Value::as_str) != Some("log") || path != "/sandbox/.saw/audit" || read_only || !claim.starts_with("saw-audit-") { return false; } + } else if !(path == "/data" || path.starts_with("/data/")) + || path[1..].split("/").any(|p| p.is_empty() || p == "." || p == "..") + || path.chars().any(|c| c.is_control()) { return false; } + if paths.iter().any(|p| p == path || path.starts_with(&format!("{p}/")) || p.starts_with(&format!("{path}/"))) { return false; } + paths.push(path.to_string()); + claims.push(claim.to_string()); + } + true +} +#[cfg(test)] +mod agent_driver_tests { + use super::*; + use serde_json::json; + + fn admin() -> HashMap { + HashMap::from([("kind".into(), "user".into()), ("provider".into(), "mtls".into()), + ("roles".into(), "viewer, openshell-admin".into())]) + } + fn operator() -> HashMap { + HashMap::from([("kind".into(), "operator".into()), ("provider".into(), "mtls".into())]) + } + fn gate_and_audit() -> Value { + json!({"podman":{"mounts":[ + {"type":"volume", "source":"saw-gate-default-notebook-12345678", + "target":"/sandbox/.saw/trusted", "read_only":true}, + {"type":"volume", "source":"saw-audit-default-notebook-12345678", + "target":"/sandbox/.saw/audit", "read_only":false} + ]}}) + } + fn with_harness() -> Value { + let mut config = gate_and_audit(); + config["podman"]["mounts"].as_array_mut().unwrap().insert(0, + json!({"type":"volume", "source":"saw-harness-default-notebook-12345678", + "target":"/sandbox/harness", "read_only":true})); + config + } + #[test] + fn rejects_unresolved_workload_templates() { + for key in ["workloadTemplate", "workload_template"] { + let mut operation = json!({}); + operation[key] = json!("stored-template"); + assert!(!validate_sandbox_driver_config(&operation, &admin()).allowed); + operation[key] = json!(""); + assert!(validate_sandbox_driver_config(&operation, &HashMap::new()).allowed); + } + for key in ["driverConfig", "driver_config"] { + let mut operation = json!({"spec":{"template":{}}}); + operation["spec"]["template"][key] = with_harness(); + assert!(validate_sandbox_driver_config(&operation, &admin()).allowed); + assert!(!validate_sandbox_driver_config(&operation, &HashMap::new()).allowed); + } + } + #[test] + fn accepts_exact_installer_combinations_and_empty_config() { + assert!(validate_driver_config(Some(&gate_and_audit()), &admin()).allowed); + assert!(validate_driver_config(Some(&with_harness()), &admin()).allowed); + assert!(validate_driver_config(Some(&gate_and_audit()), &operator()).allowed); + assert!(validate_driver_config(None, &HashMap::new()).allowed); + assert!(validate_driver_config(Some(&json!({})), &HashMap::new()).allowed); + } + #[test] + fn rejects_oidc_admin_and_nonadmin_mtls() { + let mut principal = admin(); + principal.insert("provider".into(), "oidc".into()); + assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed); + principal = admin(); + principal.insert("roles".into(), "viewer".into()); + assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed); + principal = admin(); + principal.insert("kind".into(), "service".into()); + assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed); + principal = operator(); + principal.insert("provider".into(), "oidc".into()); + assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed); + principal = operator(); + principal.insert("kind".into(), "user".into()); + assert!(!validate_driver_config(Some(&gate_and_audit()), &principal).allowed); + } + #[test] + fn rejects_wrong_source_target_type_and_mode() { + for index in 0..2 { + for (key, value) in [("type", json!("bind")), ("source", json!("other")), + ("source", json!("saw-gate-")), ("target", json!("/other")), + ("read_only", json!("wrong"))] { + let mut config = gate_and_audit(); + config["podman"]["mounts"][index][key] = value; + assert!(!validate_driver_config(Some(&config), &admin()).allowed, "accepted {config}"); + } + } + let mut config = gate_and_audit(); + config["podman"]["mounts"][0]["read_only"] = json!(false); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + config = gate_and_audit(); + config["podman"]["mounts"][1]["read_only"] = json!(true); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + } + #[test] + fn rejects_other_drivers_options_and_mount_combinations() { + for config in [json!({"docker":{"mounts":[]}}), json!({"podman":{"mounts":[]}}), + json!({"podman":{}}), json!({"podman":{"privileged":true}})] { + assert!(!validate_driver_config(Some(&config), &admin()).allowed, "accepted {config}"); + } + let mut config = gate_and_audit(); + config["podman"]["privileged"] = json!(true); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + config = gate_and_audit(); + config["podman"]["mounts"][0]["options"] = json!(["rw"]); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + config = gate_and_audit(); + config["podman"]["mounts"].as_array_mut().unwrap().swap(0, 1); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + config = gate_and_audit(); + config["podman"]["mounts"].as_array_mut().unwrap().pop(); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + config = with_harness(); + config["podman"]["mounts"].as_array_mut().unwrap().push(json!(null)); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + } + #[test] + fn rejects_malformed_envelopes() { + for config in [json!(null), json!(true), json!("bad"), json!([]), + json!({"podman":null}), json!({"podman":{"mounts":null}}), + json!({"podman":{"mounts":[null]}})] { + assert!(!validate_driver_config(Some(&config), &admin()).allowed, "accepted {config}"); + } + } + fn k8s() -> Value { + json!({"kubernetes":{ + "volumes":[{"name":"saw-workspace","persistent_volume_claim":{"claim_name":"alice-workspace","read_only":false}}], + "containers":{"agent":{"volume_mounts":[{"name":"saw-workspace","mount_path":"/sandbox/workspace","read_only":false}],"resources":{"requests":{"cpu":"250m","memory":"512Mi"}}}} + }}) + } + #[test] + fn kubernetes_tool_gate_requires_fixed_pair_and_admin() { + let mut config = k8s(); + let driver = &mut config["kubernetes"]; + for (name, sub, path, ro) in [("saw-gate","bundle","/sandbox/.saw/trusted",true), + ("saw-audit","log","/sandbox/.saw/audit",false)] { + driver["volumes"].as_array_mut().unwrap().push(json!({"name":name,"persistent_volume_claim":{"claim_name":format!("{name}-test"),"read_only":ro}})); + driver["containers"]["agent"]["volume_mounts"].as_array_mut().unwrap().push(json!({"name":name,"mount_path":path,"sub_path":sub,"read_only":ro})); + } + assert!(validate_driver_config(Some(&config), &admin()).allowed); + for (pointer, value) in [ + ("/kubernetes/containers/agent/volume_mounts/1/sub_path",json!("..")), + ("/kubernetes/containers/agent/volume_mounts/2/mount_path",json!("/etc")), + ("/kubernetes/volumes/2/persistent_volume_claim/claim_name",json!("workspace")), + ] { + let mut bad = config.clone(); + *bad.pointer_mut(pointer).unwrap() = value; + assert!(!validate_driver_config(Some(&bad), &admin()).allowed); + } + config["kubernetes"]["volumes"][1]["persistent_volume_claim"]["read_only"] = json!(false); + config["kubernetes"]["containers"]["agent"]["volume_mounts"][1]["read_only"] = json!(false); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + } + #[test] + fn kubernetes_requires_admin_and_allows_only_pvc_shape() { + let mut config = k8s(); + assert!(validate_driver_config(Some(&config), &admin()).allowed); + assert!(!validate_driver_config(Some(&config), &HashMap::new()).allowed); + let mut oidc = admin(); oidc.insert("provider".into(), "oidc".into()); + assert!(!validate_driver_config(Some(&config), &oidc).allowed); + config["kubernetes"]["volumes"].as_array_mut().unwrap().push(json!({"name":"data-0","persistent_volume_claim":{"claim_name":"alice-data","read_only":true}})); + config["kubernetes"]["containers"]["agent"]["volume_mounts"].as_array_mut().unwrap().push(json!({"name":"data-0","mount_path":"/data/models","read_only":true})); + assert!(validate_driver_config(Some(&config), &admin()).allowed); + for path in ["/etc", "/sandbox/plugin", "/data/../etc", "/data//bad", "/data/x/", "/data/./x", "/data/x\n"] { + let mut bad = config.clone(); + bad["kubernetes"]["containers"]["agent"]["volume_mounts"][1]["mount_path"] = json!(path); + assert!(!validate_driver_config(Some(&bad), &admin()).allowed); + } + config["kubernetes"]["containers"]["agent"]["volume_mounts"][1]["read_only"] = json!(false); + assert!(!validate_driver_config(Some(&config), &admin()).allowed); + } + #[test] + fn kubernetes_rejects_privilege_paths_and_malformed_options() { + for (pointer, value) in [ + ("/kubernetes/volumes/0", json!({"name":"saw-workspace","host_path":{"path":"/"}})), + ("/kubernetes/volumes/0/persistent_volume_claim", json!({"claim_name":"alice-workspace"})), + ("/kubernetes/containers/agent", json!({"volume_mounts":[],"resources":{},"security_context":{"privileged":true}})), + ("/kubernetes/containers", json!({"agent":{},"extra":{}})), + ("/kubernetes/volumes", json!([])), + ("/kubernetes/containers/agent/volume_mounts/0/mount_path", json!("/sandbox")), + ("/kubernetes/volumes/0/persistent_volume_claim/claim_name", json!("../secret")), + ("/kubernetes/containers/agent/resources", json!({"requests":{"cpu":"0"}})), + ("/kubernetes/containers/agent/resources", json!({"requests":{"gpu":"1"}})), + ("/kubernetes/containers/agent/resources", json!({"requests":{"cpu":1}})) + ] { + let mut bad=k8s(); *bad.pointer_mut(pointer).unwrap()=value; + assert!(!validate_driver_config(Some(&bad), &admin()).allowed, "accepted {bad}"); + } + for pod in [json!({}),json!({"runtime_class_name":""}),json!({"host_network":true}),json!({"runtime_class_name":"custom-runtime"})] { + let mut bad=k8s(); bad["kubernetes"]["pod"]=pod; + assert!(!validate_driver_config(Some(&bad), &admin()).allowed); + } + let mut bad=k8s(); bad["podman"]=json!({}); + assert!(!validate_driver_config(Some(&bad), &admin()).allowed); + } +} diff --git a/image-builder-charts/helm/governance-interceptor-image/files/saw-extension/integration.patch b/image-builder-charts/helm/governance-interceptor-image/files/saw-extension/integration.patch new file mode 100644 index 000000000..631a1fdb7 --- /dev/null +++ b/image-builder-charts/helm/governance-interceptor-image/files/saw-extension/integration.patch @@ -0,0 +1,37 @@ +--- a/src/main.rs ++++ b/src/main.rs +@@ -1,5 +1,7 @@ + // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + // SPDX-License-Identifier: Apache-2.0 ++ ++mod saw_driver_config; + + mod policy_hash; + mod proto_json; +@@ -373,6 +375,12 @@ + Self::patch_create_sandbox(&operation, &policy_state) + } + ("CreateSandbox", interceptor_evaluation::Phase::Validate(_)) => { ++ let gate = saw_driver_config::validate_sandbox_driver_config( ++ &operation, &evaluation.principal, ++ ); ++ if !gate.allowed { ++ return Ok(gate); ++ } + Ok(validate_create_sandbox( + &operation, + &profile_state.ids, +@@ -969,13 +977,6 @@ + + let profile_hash = deterministic_profile_hash(&profile)?; + let profile_signature = policy_signer.sign_profile(&profile.id, &profile_hash)?; +- profile +- .annotations +- .insert(PROFILE_HASH_ANNOTATION.to_string(), profile_hash); +- profile.annotations.insert( +- PROFILE_SIGNATURE_KID_ANNOTATION.to_string(), +- policy_signer.kid().to_string(), +- ); + profile + .annotations + .insert(PROFILE_SIGNATURE_ANNOTATION.to_string(), profile_signature); diff --git a/image-builder-charts/helm/governance-interceptor-image/templates/buildconfig.yaml b/image-builder-charts/helm/governance-interceptor-image/templates/buildconfig.yaml index 42d7ecf46..d68258bb0 100644 --- a/image-builder-charts/helm/governance-interceptor-image/templates/buildconfig.yaml +++ b/image-builder-charts/helm/governance-interceptor-image/templates/buildconfig.yaml @@ -1,9 +1,12 @@ apiVersion: build.openshift.io/v1 kind: BuildConfig metadata: - name: governance-interceptor + name: {{ .Values.build.name | quote }} spec: source: + configMaps: + - configMap: {name: {{ .Values.build.name }}-source} + destinationDir: saw-extension type: Dockerfile dockerfile: | {{ .Files.Get "files/Dockerfile" | indent 6 }} @@ -38,4 +41,9 @@ spec: resources: limits: memory: 4Gi + {{- if .Values.build.configChangeTrigger }} + triggers: + - type: ConfigChange + {{- else }} triggers: [] + {{- end }} diff --git a/image-builder-charts/helm/governance-interceptor-image/templates/source.yaml b/image-builder-charts/helm/governance-interceptor-image/templates/source.yaml new file mode 100644 index 000000000..e6a84cf57 --- /dev/null +++ b/image-builder-charts/helm/governance-interceptor-image/templates/source.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.build.name }}-source +data: +{{ (.Files.Glob "files/saw-extension/*").AsConfig | indent 2 }} diff --git a/image-builder-charts/helm/governance-interceptor-image/values.yaml b/image-builder-charts/helm/governance-interceptor-image/values.yaml index 50e1d0129..afcdd08ce 100644 --- a/image-builder-charts/helm/governance-interceptor-image/values.yaml +++ b/image-builder-charts/helm/governance-interceptor-image/values.yaml @@ -4,6 +4,10 @@ source: ref: "v0.1.6" build: + # For GitOps, give each new build recipe its own BuildConfig name. + name: governance-interceptor + # Default preserves the existing explicit Make build command. + configChangeTrigger: false contextDir: "examples/governance-interceptor" # OpenShell 0.1.x needs Rust 1.94 (rust-version in its Cargo.toml). builderImage: "rust:1.94-bookworm" diff --git a/image-builder-charts/helm/openshift-home-access/Chart.yaml b/image-builder-charts/helm/openshift-home-access/Chart.yaml new file mode 100644 index 000000000..aa2bb40b0 --- /dev/null +++ b/image-builder-charts/helm/openshift-home-access/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: openshift-home-access +version: 0.1.0 +description: Add directory traversal for an OpenShift-assigned UID to an existing pinned workspace image. diff --git a/image-builder-charts/helm/openshift-home-access/templates/build.yaml b/image-builder-charts/helm/openshift-home-access/templates/build.yaml new file mode 100644 index 000000000..95578e9cc --- /dev/null +++ b/image-builder-charts/helm/openshift-home-access/templates/build.yaml @@ -0,0 +1,25 @@ +{{- range $i := .Values.images }} +{{- if or (not (regexMatch "^[a-z0-9][a-z0-9./:_-]+@sha256:[0-9a-f]{64}$" $i.from)) (not (regexMatch "^[A-Za-z0-9_-]+(:[A-Za-z0-9_-]+)?$" $i.imageUser)) }}{{ fail "a digest-pinned source and original image user are required" }}{{- end }} +--- +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: {name: {{ $i.name }}, namespace: {{ $.Release.Namespace }}} +--- +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: {name: {{ $i.name }}, namespace: {{ $.Release.Namespace }}} +spec: + source: + type: Dockerfile + dockerfile: | + FROM {{ $i.from }} + USER 0 + RUN test -d /sandbox && chmod o+x /sandbox + USER {{ $i.imageUser }} + {{- with $.Values.buildArchitecture }} + nodeSelector: {kubernetes.io/arch: {{ . | quote }}} + {{- end }} + strategy: {type: Docker, dockerStrategy: {}} + output: {to: {kind: ImageStreamTag, name: "{{ $i.name }}:traverse"}} + triggers: {{ ternary "[{type: ConfigChange}]" "[]" $.Values.configChangeTrigger }} +{{- end }} diff --git a/image-builder-charts/helm/openshift-home-access/values.yaml b/image-builder-charts/helm/openshift-home-access/values.yaml new file mode 100644 index 000000000..e5b0cbc09 --- /dev/null +++ b/image-builder-charts/helm/openshift-home-access/values.yaml @@ -0,0 +1,3 @@ +images: [] +configChangeTrigger: true +buildArchitecture: "" diff --git a/kubernetes-runtime/Containerfile b/kubernetes-runtime/Containerfile new file mode 100644 index 000000000..e50b368da --- /dev/null +++ b/kubernetes-runtime/Containerfile @@ -0,0 +1,22 @@ +FROM registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c AS generate +USER 0 +WORKDIR /build +COPY kubernetes-runtime/build-sdk.py kubernetes-runtime/sdk-lock.json /build/ +RUN pip install --no-cache-dir grpcio-tools==1.80.0 grpcio==1.80.0 protobuf==6.33.5 && python3 build-sdk.py + +FROM registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c +USER 0 +RUN pip install --no-cache-dir grpcio==1.80.0 protobuf==6.33.5 PyYAML==6.0.3 +RUN command -v openssl && command -v shred && command -v stat +WORKDIR /opt/saw +COPY --from=generate /build/python/openshell/_proto /opt/saw/sdk/openshell/_proto +COPY charts/openshell-saw/files/installer/saw_core.py charts/openshell-saw/files/installer/saw_core.py +COPY charts/openshell-saw/files/installer/saw_admin.py charts/openshell-saw/files/installer/saw_admin.py +COPY charts/openshell-saw/files/installer/tool-actions.yaml charts/openshell-saw/files/installer/tool-gate.mjs charts/openshell-saw/files/installer/tool-gate-plugin.mjs charts/openshell-saw/files/installer/ +COPY charts/openshell-saw/files/provider-profiles/ charts/openshell-saw/files/provider-profiles/ +COPY charts/saw-users/files/workspace.py /opt/saw/kubernetes-runtime/saw_workspace.py +COPY kubernetes-runtime/saw_kubernetes kubernetes-runtime/saw_kubernetes +ENV PYTHONPATH=/opt/saw/kubernetes-runtime:/opt/saw/sdk PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +RUN python3 -c 'from openshell._proto import openshell_pb2_grpc; from saw_kubernetes import serve' +USER 1001 +ENTRYPOINT ["python3", "-m", "saw_kubernetes.serve"] diff --git a/kubernetes-runtime/Containerfile.dockerignore b/kubernetes-runtime/Containerfile.dockerignore new file mode 100644 index 000000000..71d42dc95 --- /dev/null +++ b/kubernetes-runtime/Containerfile.dockerignore @@ -0,0 +1,24 @@ +# This build needs only the adapter and its canonical shared sources. +** +!kubernetes-runtime/ +!kubernetes-runtime/Containerfile +!kubernetes-runtime/build-sdk.py +!kubernetes-runtime/sdk-lock.json +!kubernetes-runtime/saw_kubernetes/ +!kubernetes-runtime/saw_kubernetes/*.py +!kubernetes-runtime/saw_kubernetes/*.js +!kubernetes-runtime/saw_kubernetes/*.sh +!charts/ +!charts/openshell-saw/ +!charts/openshell-saw/files/ +!charts/openshell-saw/files/installer/ +!charts/openshell-saw/files/installer/saw_core.py +!charts/openshell-saw/files/installer/saw_admin.py +!charts/openshell-saw/files/installer/tool-actions.yaml +!charts/openshell-saw/files/installer/tool-gate.mjs +!charts/openshell-saw/files/installer/tool-gate-plugin.mjs +!charts/openshell-saw/files/provider-profiles/ +!charts/openshell-saw/files/provider-profiles/*.yaml +!charts/saw-users/ +!charts/saw-users/files/ +!charts/saw-users/files/workspace.py diff --git a/kubernetes-runtime/build-sdk.py b/kubernetes-runtime/build-sdk.py new file mode 100644 index 000000000..069a2f433 --- /dev/null +++ b/kubernetes-runtime/build-sdk.py @@ -0,0 +1,19 @@ +"""Generate wire bindings from verified stock OpenShell source.""" +import hashlib +import json +from pathlib import Path +import subprocess +import sys +from urllib.request import urlopen + +revision = "305c8b3d7bbdb97a6f2f7264c3ad43b0087c307f" +for name, expected in json.loads(Path("/build/sdk-lock.json").read_text()).items(): + with urlopen(f"https://raw.githubusercontent.com/opendatahub-io/openshell/{revision}/{name}", timeout=30) as response: + content = response.read() + if hashlib.sha256(content).hexdigest() != expected: + raise ValueError("stock protobuf source checksum mismatch") + path = Path(name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) +Path("python/openshell/_proto").mkdir(parents=True) +subprocess.run([sys.executable, "tasks/scripts/generate_python_proto.py"], check=True) diff --git a/kubernetes-runtime/saw_kubernetes/__init__.py b/kubernetes-runtime/saw_kubernetes/__init__.py new file mode 100644 index 000000000..2268601d3 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/__init__.py @@ -0,0 +1,6 @@ +"""Experimental Kubernetes profile compiler; not a replacement VM installer.""" + +from .compiler import ContractError, compile_profile, compile_profile_files +from .reconcile import Reconciler, RpcError + +__all__ = ["ContractError", "compile_profile", "compile_profile_files", "Reconciler", "RpcError"] diff --git a/kubernetes-runtime/saw_kubernetes/__main__.py b/kubernetes-runtime/saw_kubernetes/__main__.py new file mode 100644 index 000000000..7c5d2c877 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/__main__.py @@ -0,0 +1,30 @@ +"""Compile only: no credentials, gateway connection or Kubernetes operations.""" + +import argparse +import json +from pathlib import Path +import sys + +import yaml + +from .compiler import ContractError, compile_profile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", required=True) + parser.add_argument("--bindings", required=True, help="Administrator YAML/JSON bindings file") + args = parser.parse_args() + try: + config = yaml.safe_load(Path(args.bindings).read_text()) + result = compile_profile(args.profile, config) + except (ContractError, OSError, yaml.YAMLError) as exc: + message = str(exc) if isinstance(exc, ContractError) else "cannot read input documents" + print(message, file=sys.stderr) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kubernetes-runtime/saw_kubernetes/adapters.py b/kubernetes-runtime/saw_kubernetes/adapters.py new file mode 100644 index 000000000..11535a3ed --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/adapters.py @@ -0,0 +1,164 @@ +"""Opt-in live adapters. Importing this module makes no connection or mutation.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +from urllib.parse import urlsplit + +from .compiler import ContractError, OWNER, digest +from .reconcile import RpcError + +RPC_METHODS = { + "GetGatewayInfo", "GetWorkspace", "CreateWorkspace", "ListWorkspaceMembers", + "AddWorkspaceMember", "GetProviderProfile", "GetProvider", "CreateProvider", "GetSandbox", + "CreateSandbox", "StartSandbox", "StopSandbox", +} + + +class GrpcGateway: + """Use generated Python stubs from the pinned OpenShell source, over mTLS. + + No shell commands, insecure transport, server-name overrides or automatic + mutation retry. The caller owns lifecycle and credential rotation. + """ + + def __init__(self, endpoint, ca_file, certificate_file, key_file, *, timeout=20): + if not isinstance(endpoint, str) or any(c.isspace() for c in endpoint): + raise ContractError("gateway endpoint must be an HTTPS authority") + try: + parsed = urlsplit(endpoint) + port = parsed.port + except ValueError: + raise ContractError("gateway endpoint must have a valid host and port") from None + if (parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password + or parsed.path not in {"", "/"} or parsed.query or parsed.fragment or port == 0): + raise ContractError("gateway endpoint must be an HTTPS authority without credentials or path") + if not isinstance(timeout, (float, int)) or isinstance(timeout, bool) or not 0 < timeout <= 60: + raise ContractError("RPC timeout must be between zero and 60 seconds") + try: + import grpc + from google.protobuf.json_format import MessageToDict, ParseDict, ParseError + from openshell._proto import openshell_pb2, openshell_pb2_grpc + except ImportError: + raise ContractError("install the generated Python SDK from the pinned OpenShell source") from None + try: + roots = Path(ca_file).read_bytes() + cert = Path(certificate_file).read_bytes() + key = Path(key_file).read_bytes() + if not roots or not cert or not key: + raise ValueError() + except (OSError, ValueError): + raise ContractError("gateway CA and private client identity files are required") from None + self.grpc, self.pb2 = grpc, openshell_pb2 + self.parse, self.serialize = ParseDict, MessageToDict + self.parse_error = ParseError + self.timeout = timeout + target = parsed.netloc + self.channel = grpc.secure_channel(target, grpc.ssl_channel_credentials(roots, key, cert)) + self.stub = openshell_pb2_grpc.OpenShellStub(self.channel) + + def call(self, method, request): + if method not in RPC_METHODS: + raise ContractError("RPC is outside the reconciler allowlist") + try: + message = self.parse(request, getattr(self.pb2, method + "Request")(), ignore_unknown_fields=False) + except (self.parse_error, ValueError, TypeError, AttributeError): + raise ContractError("request does not match the pinned OpenShell protobuf API") from None + try: + response = getattr(self.stub, method)(message, timeout=self.timeout) + except self.grpc.RpcError as exc: + raise RpcError(exc.code().name) from None + return self.serialize(response, preserving_proto_field_name=True) + + def close(self): + self.channel.close() + + +class OcReader: + """Read only namespaces/PVCs/RuntimeClasses using existing verified oc auth.""" + + def __init__(self, executable="oc", *, timeout=20): + self.executable, self.timeout = executable, timeout + + def __call__(self, kind, name, namespace=None): + if kind not in {"namespace", "persistentvolumeclaim", "runtimeclass"}: + raise ContractError("resource lookup is outside the verifier allowlist") + command = [self.executable, "get", kind, name, "-o", "json", "--request-timeout=15s"] + if namespace: + command.extend(["--namespace", namespace]) + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=self.timeout, check=True) + return json.loads(result.stdout) + except (subprocess.SubprocessError, OSError, ValueError): + raise ContractError("cannot verify current Kubernetes resource identity") from None + + +class KubernetesResourceVerifier: + """Verify identities and default stock driver attachment approval labels. + + `verify_boundary` is mandatory: integration must separately prove its native + CNI/admission enforcement prerequisite. Checking a label is not such proof. + The OpenShell driver still performs its own admission at create/restart. + """ + + def __init__(self, read_json, verify_boundary): + if not callable(read_json) or not callable(verify_boundary): + raise ContractError("resource reader and boundary verifier are required") + self.read, self.verify_boundary = read_json, verify_boundary + + @staticmethod + def _identity(obj, uid): + metadata = obj.get("metadata", {}) + if metadata.get("uid") != uid or metadata.get("deletionTimestamp"): + raise ContractError("Kubernetes resource replaced or deleting") + return metadata + + def verify_shutdown(self, plan): + """Shutdown checks namespace identity, not renewed attachment permission. + + The reconciler still checks authenticated gateway identity and retained + workspace/sandbox IDs. This method grants no create/start capability. + """ + namespace = self.read("namespace", plan["namespace"]) + meta = self._identity(namespace, plan["namespaceUID"]) + if meta.get("labels", {}).get(OWNER) != plan["ownerID"]: + raise ContractError("runtime namespace belongs to another installation") + + def __call__(self, plan): + self.verify_boundary(plan) + self.verify_attachments(plan) + + def verify_attachments(self, plan): + """Frontend-owned Running passes already verified the boundary before opening the gateway.""" + self.verify_shutdown(plan) + for reference in plan["claims"]: + obj = self.read("persistentvolumeclaim", reference["name"], plan["namespace"]) + meta = self._identity(obj, reference["uid"]) + labels = meta.get("labels", {}) + if (labels.get("openshell.ai/sandbox-attachable") != "true" + or labels.get("openshell.ai/sandbox-attachable-workspace") != reference["workspace"] + or labels.get(OWNER) != plan["ownerID"]): + raise ContractError("PVC is not approved for this workspace and installation") + if meta.get("ownerReferences"): + raise ContractError("retained PVC must not have garbage-collecting owner references") + sync = meta.get("annotations", {}).get("argocd.argoproj.io/sync-options", "") + options = {s.strip() for s in sync.split(",")} + required = {"Prune=false", "Delete=false"} + conflicting = {s for s in options if s.startswith(("Prune=", "Delete="))} - required + if not required <= options or conflicting: + raise ContractError("retained PVC requires explicit Argo prune and application-deletion protection") + if obj.get("spec", {}).get("volumeMode", "Filesystem") != "Filesystem": + raise ContractError("workspace adapter requires filesystem PVCs") + if obj.get("status", {}).get("phase") not in {"Pending", "Bound"}: + raise ContractError("PVC is not available for binding") + runtime = plan["runtimeClass"] + if runtime["name"]: + obj = self.read("runtimeclass", runtime["name"]) + meta = self._identity(obj, runtime["uid"]) + if meta.get("labels", {}).get("openshell.ai/sandbox-attachable") != "true": + raise ContractError("RuntimeClass has not been approved") + placement = {k: obj.get(k) for k in ("handler", "overhead", "scheduling")} + if obj.get("handler") != "kata" or digest(placement) != runtime.get("sha256"): + raise ContractError("RuntimeClass handler, scheduling or overhead differs from approval") diff --git a/kubernetes-runtime/saw_kubernetes/admin_session.py b/kubernetes-runtime/saw_kubernetes/admin_session.py new file mode 100644 index 000000000..d34d71cb9 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/admin_session.py @@ -0,0 +1,211 @@ +"""Kubernetes trust-bundle adapter for the VM's temporary admin identity. + +Only public CA certificates enter the named Secret. resourceVersion provides +compare-and-swap; concurrent controller/relay sessions preserve each other's +trust. Dead-Pod and expired sessions are removed at the next trust update. +""" +import base64 +from contextlib import contextmanager +import copy +import fcntl +import json +import os +from pathlib import Path +import re +import ssl +import time +import uuid + +from .adapters import GrpcGateway +from .compiler import ContractError +from .controller import ApiError +from .core import admin +from .reconcile import RpcError + +SESSIONS = "saw.redhat.com/admin-sessions" +OWNER = "saw.redhat.com/runtime-owner" +NAME = re.compile(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?\Z") +OPERATOR_SUBJECT = "/O=openshell/OU=operator/CN=saw-installer" + + +def encoded(raw): + return base64.b64encode(raw).decode("ascii") + + +def bundle(base, certificates): + return base.rstrip()+b"\n"+b"".join(certificates[key].rstrip()+b"\n" for key in sorted(certificates)) + + +class TrustBundle: + def __init__(self, request, namespace, trusted_namespace, gateway, owner_id, base_ca, pod, pod_uid, *, now=time.time): + if not all(isinstance(v, str) and NAME.fullmatch(v) for v in (namespace, trusted_namespace, gateway, pod)): + raise ContractError("invalid temporary administrator scope") + try: + for value in (owner_id, pod_uid): + if str(uuid.UUID(value)) != value or uuid.UUID(value).int == 0: + raise ValueError() + except (ValueError, TypeError, AttributeError): + raise ContractError("invalid temporary administrator identity") from None + self.request, self.now = request, now + self.namespace, self.trusted_namespace = namespace, trusted_namespace + self.name, self.owner_id = gateway+"-client-ca", owner_id + self.base, self.pod, self.pod_uid = base_ca, pod, pod_uid + self.path = f"/api/v1/namespaces/{namespace}/secrets/{self.name}" + self.uid = None + + def _pod(self, holder): + try: + return self.request("GET", f"/api/v1/namespaces/{self.trusted_namespace}/pods/{holder['pod']}") + except ApiError as exc: + if exc.code == 404: return None + raise + + def live(self, holder): + pod = self._pod(holder) + meta = (pod or {}).get("metadata", {}) + return (meta.get("uid") == holder["uid"] and not meta.get("deletionTimestamp") + and (pod or {}).get("status", {}).get("phase") == "Running") + + def wait_for_holder(self, *, timeout=30, sleep=time.sleep, monotonic=time.monotonic): + """A Job process can start before kubelet publishes its Running phase. + + Wait only for this exact, nondeleting Pod while it is Pending. Do not + publish trust for it yet or relax live() for cleanup/publication checks. + """ + deadline = monotonic() + timeout + while True: + pod = self._pod({"pod": self.pod}) or {} + meta, phase = pod.get("metadata", {}), pod.get("status", {}).get("phase") + if meta.get("uid") != self.pod_uid or meta.get("deletionTimestamp"): + raise ContractError("administrator session holder missing, replaced or deleting") + if phase == "Running": + return + if phase != "Pending": + raise ContractError("administrator session holder is not starting or running") + remaining = deadline - monotonic() + if remaining <= 0: + raise ContractError("administrator session holder did not become Running before the deadline") + sleep(min(0.5, remaining)) + + def read(self): + obj = self.request("GET", self.path) + meta = obj.get("metadata", {}) + data = obj.get("data", {}) + if (meta.get("name") != self.name or meta.get("namespace") != self.namespace + or meta.get("labels", {}).get(OWNER) != self.owner_id + or meta.get("deletionTimestamp") or meta.get("ownerReferences") + or not meta.get("uid") or not meta.get("resourceVersion") + or self.uid is not None and self.uid != meta["uid"] + or obj.get("immutable") is not False or obj.get("type") != "Opaque" + or data.get("base.crt") != encoded(self.base)): + raise ContractError("gateway client trust is foreign, replaced or immutable") + self.uid = meta["uid"] + try: + sessions = json.loads(meta.get("annotations", {}).get(SESSIONS, "{}")) + if not isinstance(sessions, dict) or len(sessions) > 16: + raise ValueError() + certs = {} + for sid, holder in sessions.items(): + if str(uuid.UUID(sid)) != sid or set(holder) != {"pod", "uid", "expires"}: + raise ValueError() + if (not NAME.fullmatch(holder["pod"]) or str(uuid.UUID(holder["uid"])) != holder["uid"] + # Allow modest node clock skew in metadata. TLS still + # enforces the certificate's actual one-day validity. + or type(holder["expires"]) is not int or holder["expires"] > self.now()+86400+300): + raise ValueError() + certs[sid] = base64.b64decode(data["session-"+sid+".crt"], validate=True) + if not re.fullmatch(rb"-----BEGIN CERTIFICATE-----[A-Za-z0-9+/=\r\n]+-----END CERTIFICATE-----\s*", certs[sid]): + raise ValueError() + ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=certs[sid].decode("ascii")) + if set(data) != {"base.crt", "ca.crt", *("session-"+sid+".crt" for sid in sessions)}: + raise ValueError() + if data["ca.crt"] != encoded(bundle(self.base, certs)): + raise ValueError() + except (ValueError, TypeError, KeyError, AttributeError, ssl.SSLError): + raise ContractError("gateway temporary trust publication is malformed") from None + return obj, sessions, certs + + def update(self, *, add=None, remove=None, clear_holder=False): + for _ in range(8): + obj, sessions, certs = self.read() + before = copy.deepcopy(sessions) + for sid, holder in list(sessions.items()): + own = holder["uid"] == self.pod_uid and holder["pod"] == self.pod + if (clear_holder and own) or holder["expires"] <= self.now() or not self.live(holder): + del sessions[sid], certs[sid] + if remove and remove in sessions: + if sessions[remove]["uid"] != self.pod_uid or sessions[remove]["pod"] != self.pod: + raise ContractError("cannot withdraw another administrator session") + del sessions[remove], certs[remove] + if add: + holder = {"pod": self.pod, "uid": self.pod_uid, "expires": add["expires"]} + if not self.live(holder) or (add["id"] not in sessions and len(sessions) >= 16): + raise ContractError("administrator session holder unavailable or limit exceeded") + if add["id"] in sessions and sessions[add["id"]] != holder: + raise ContractError("administrator session identity collision") + sessions[add["id"]], certs[add["id"]] = holder, add["ca"].encode() + if sessions == before and not add: + return + obj["metadata"].setdefault("annotations", {})[SESSIONS] = json.dumps(sessions, sort_keys=True) + obj["data"] = {"base.crt": encoded(self.base), "ca.crt": encoded(bundle(self.base, certs)), + **{"session-"+sid+".crt": encoded(cert) for sid, cert in certs.items()}} + try: + self.request("PUT", self.path, obj) + return + except ApiError as exc: + if exc.code != 409: raise + raise ContractError("gateway temporary trust update conflicted") + + +class GatewaySessions: + def __init__(self, trust, endpoint, runtime="/run/saw-admin", *, gateway_factory=GrpcGateway): + self.trust, self.endpoint, self.runtime = trust, endpoint, runtime + self.gateway_factory = gateway_factory + + @contextmanager + def _exclusive(self): + admin.require_memory(self.runtime) + fd = os.open(Path(self.runtime)/"session.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ContractError("another administrator operation owns this Pod's session") from None + try: + self.trust.update(clear_holder=True) + finally: + # Also remove private files if API recovery is unavailable. + admin.remove_abandoned(self.runtime) + self.trust.wait_for_holder() + yield + finally: + os.close(fd) + + @contextmanager + def open(self): + with self._exclusive(), admin.identity( + self.trust.base, self.runtime, subject=OPERATOR_SUBJECT) as identity: + gateway = None + try: + # Cleanup also runs after an ambiguous API response: a write + # may have committed even when its response was lost. + self.trust.update(add=identity) + def accepted(): + nonlocal gateway + if gateway is not None: + gateway.close() + tls = identity["tls"] + gateway = self.gateway_factory(self.endpoint, tls/"ca.crt", tls/"client/tls.crt", tls/"client/tls.key", timeout=3) + try: + gateway.call("GetGatewayInfo", {}) + return True + except RpcError as exc: + if exc.code not in {"UNAVAILABLE", "UNAUTHENTICATED"}: raise + return False + admin.wait_for_trust(accepted) + gateway.timeout = 20 + yield gateway + finally: + if gateway is not None: + gateway.close() + self.trust.update(remove=identity["id"]) diff --git a/kubernetes-runtime/saw_kubernetes/audit.py b/kubernetes-runtime/saw_kubernetes/audit.py new file mode 100644 index 000000000..a8367e966 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/audit.py @@ -0,0 +1,31 @@ +"""Emit candidate boundary hashes for administrator review using the active oc login.""" +import argparse +import json +import subprocess +from pathlib import Path + +from .boundary import fingerprint, objects, verify + + +def capture(config, get): + approved = {} + for key, (path, field) in objects(config).items(): + obj = get(path) + approved[key] = {"uid": obj["metadata"]["uid"], "sha256": fingerprint(obj[field])} + result = dict(config, approved=approved) + verify(result, result, get) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("configuration", type=Path) + args = parser.parse_args() + + def get(path): + return json.loads(subprocess.check_output(["oc", "get", "--raw", path, "--request-timeout=20s"])) + print(json.dumps(capture(json.loads(args.configuration.read_text()), get), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/kubernetes-runtime/saw_kubernetes/boundary.py b/kubernetes-runtime/saw_kubernetes/boundary.py new file mode 100644 index 000000000..ea2c1c667 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/boundary.py @@ -0,0 +1,157 @@ +"""Read-only checks of the administrator-approved OpenShift installation.""" +import hashlib +import json + +import yaml + +from . import provider as governed + + +class BoundaryError(ValueError): + pass + + +def fingerprint(value): + data = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(data.encode()).hexdigest() + + +def objects(config): + ns = config["namespace"] + trusted = config["trustedNamespace"] + governance = config["governanceNamespace"] + gateway = config["gateway"] + policy = f"saw-{ns}-runtime" + admission = "/apis/admissionregistration.k8s.io/v1" + apps = "/apis/apps/v1/namespaces" + net = "/apis/networking.k8s.io/v1/namespaces" + core = "/api/v1/namespaces" + placement = {} + if config.get("mode") == "kata": + placement = { + "sandboxPlacement": (f"{admission}/validatingadmissionpolicies/{policy}-kata", "spec"), + "sandboxPlacementBinding": (f"{admission}/validatingadmissionpolicybindings/{policy}-kata", "spec"), + } + return { + **placement, + "validation": (f"{admission}/validatingadmissionpolicies/{policy}", "spec"), + "podDeleteGuard": (f"{admission}/validatingadmissionpolicies/{ns}-pods-driver", "spec"), + "podDeleteBinding": (f"{admission}/validatingadmissionpolicybindings/{ns}-pods-driver", "spec"), + "secretGuard": (f"{admission}/validatingadmissionpolicies/{ns}-secrets-driver", "spec"), + "secretBinding": (f"{admission}/validatingadmissionpolicybindings/{ns}-secrets-driver", "spec"), + "serviceGuard": (f"{admission}/validatingadmissionpolicies/{ns}-services-driver", "spec"), + "serviceBinding": (f"{admission}/validatingadmissionpolicybindings/{ns}-services-driver", "spec"), + "gatewayRole": (f"/apis/rbac.authorization.k8s.io/v1/namespaces/{ns}/roles/{gateway}-gateway", "rules"), + "workloadFence": (f"{net}/{ns}/networkpolicies/openshell-sandbox-workloads", "spec"), + "supervisorFence": (f"{net}/{ns}/networkpolicies/openshell-sandbox-supervisors", "spec"), + "binding": (f"{admission}/validatingadmissionpolicybindings/{policy}", "spec"), + "network": (f"/apis/policy.networking.k8s.io/v1alpha1/adminnetworkpolicies/{policy}-egress", "spec"), + "deny": (f"{net}/{ns}/networkpolicies/{gateway}-default-deny", "spec"), + "gatewayNetwork": (f"{net}/{ns}/networkpolicies/{gateway}-gateway", "spec"), + "gateway": (f"{apps}/{ns}/statefulsets/{gateway}-gateway", "spec"), + "gatewayService": (f"{core}/{ns}/services/{gateway}-gateway", "spec"), + "gatewayConfig": (f"{core}/{ns}/configmaps/{gateway}-gateway-config", "data"), + "governance": (f"{apps}/{governance}/deployments/governance-interceptor", "spec"), + "governanceService": (f"{core}/{governance}/services/governance-interceptor", "spec"), + "governancePolicy": (f"{core}/{governance}/configmaps/governance-interceptor-policy", "data"), + "governanceProfiles": (f"{core}/{governance}/configmaps/governance-interceptor-profiles", "data"), + "proxy": (f"{apps}/{trusted}/deployments/{gateway}-credential-proxy", "spec"), + "proxyNetwork": (f"{net}/{trusted}/networkpolicies/{gateway}-credential-proxy", "spec"), + "proxyPolicy": (f"{core}/{trusted}/configmaps/{config['proxyPolicy']}", "data"), + "publicCA": (f"{core}/{ns}/configmaps/{gateway}-proxy-ca", "data"), + } + + +def complete_list(get, path): + response = get(path) + if response.get("metadata", {}).get("continue") or not isinstance(response.get("items"), list): + raise BoundaryError("IncompleteInventory") + return response["items"] + + +def excludes_namespace(policy, namespace): + """Prove disjointness only with the Kubernetes-managed, immutable name label. + + Pod labels, ordinary namespace labels, and unsupported selector forms do + not establish an isolation boundary. A missing/unknown subject overlaps. + """ + subject = policy.get("spec", {}).get("subject", {}) + if not isinstance(subject, dict): + return False + if set(subject) == {"pods"} and isinstance(subject["pods"], dict): + selector = subject["pods"].get("namespaceSelector") + elif set(subject) == {"namespaces"}: + selector = subject["namespaces"] + else: + return False + labels = selector.get("matchLabels") if isinstance(selector, dict) else None + name = labels.get("kubernetes.io/metadata.name") if isinstance(labels, dict) else None + return isinstance(name, str) and bool(name) and name != namespace + + +def verify(config, plan, get): + if (config["namespace"], config["namespaceUID"], config["ownerID"], config["mode"]) != ( + plan["namespace"], plan["namespaceUID"], plan["ownerID"], plan["mode"]): + raise BoundaryError("WrongInstallation") + refs = objects(config) + if set(config["approved"]) != set(refs): + raise BoundaryError("IncompleteApproval") + observed = {} + for key, (path, field) in refs.items(): + obj = get(path) + meta = obj["metadata"] + expected = config["approved"][key] + if (meta.get("deletionTimestamp") or meta["uid"] != expected["uid"] + or fingerprint(obj[field]) != expected["sha256"]): + raise BoundaryError("ChangedBoundary:" + key) + observed[key] = obj + for workspace in plan.get("workspaces", []): + for provider in workspace["providers"]: + credentials = provider.get("credentials") + if credentials: + policy = json.loads(observed["proxyPolicy"]["data"]["config.json"]) + kind = provider.get("type", "openai") + if kind not in {"openai", "custom-inference"}: + live = observed["governanceProfiles"]["data"].get(kind + ".yaml", "") + if yaml.safe_load(live) != governed.document(kind): + raise BoundaryError("ProviderGovernanceMismatch") + if not governed.check_policy(kind, credentials, policy, plan.get("customInferenceURL")): + raise BoundaryError("OpenAIProxyBindingMismatch" if kind == "openai" + else "ProviderProxyBindingMismatch") + checks = ["validation", "secretGuard", "serviceGuard", "podDeleteGuard"] + if config["mode"] == "kata": + import tomllib + driver = tomllib.loads(observed["gatewayConfig"]["data"]["gateway.toml"])["openshell"]["drivers"]["kubernetes"] + runtime = config.get("workloadRuntimeClass") + if (not runtime or plan.get("runtimeClass", {}).get("name", runtime) != runtime + or driver.get("default_runtime_class_name") != runtime): + raise BoundaryError("WrongWorkloadPlacement") + checks.append("sandboxPlacement") + for key in checks: + vap = observed[key] + status = vap.get("status", {}) + if (status.get("observedGeneration") != vap["metadata"]["generation"] + or "typeChecking" not in status or status["typeChecking"].get("expressionWarnings")): + raise BoundaryError("AdmissionUnavailable:" + key) + network = observed["network"] + priority = network["spec"]["priority"] + policies = complete_list(get, "/apis/policy.networking.k8s.io/v1alpha1/adminnetworkpolicies") + if any(p["metadata"]["uid"] != network["metadata"]["uid"] and p["spec"]["priority"] <= priority + and not excludes_namespace(p, config["namespace"]) + for p in policies): + raise BoundaryError("EarlierNetworkPolicy") + nodes = complete_list(get, "/api/v1/nodes") + zones = {n["metadata"]["name"] for n in nodes} + realized = {c["type"].removeprefix("Ready-In-Zone-") + for c in network.get("status", {}).get("conditions", []) + if c["type"].startswith("Ready-In-Zone-") and c["status"] == "True" + and c.get("reason") == "SetupSucceeded"} + if not zones or not zones <= realized: + raise BoundaryError("NetworkNotRealized") + for key in ("gateway", "governance", "proxy"): + obj = observed[key] + status = obj.get("status", {}) + if (status.get("observedGeneration") != obj["metadata"]["generation"] + or status.get("readyReplicas", 0) != obj["spec"]["replicas"] + or status.get("updatedReplicas", 0) != obj["spec"]["replicas"]): + raise BoundaryError("RolloutIncomplete:" + key) diff --git a/kubernetes-runtime/saw_kubernetes/compiler.py b/kubernetes-runtime/saw_kubernetes/compiler.py new file mode 100644 index 000000000..6f7f493c1 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/compiler.py @@ -0,0 +1,482 @@ +"""Translate a deliberately small SAW profile subset to pinned OpenShell RPCs. + +Only the administrator binding file may select PVCs/resources/RuntimeClasses. +No API calls, secret reads or mutations happen during compilation. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import uuid +from pathlib import Path, PurePosixPath + +from .core import shared +from . import harness +from . import provider as governed + +SOURCE_REVISION = "305c8b3d7bbdb97a6f2f7264c3ad43b0087c307f" +OWNER = "saw.redhat.com/runtime-owner" +REVISION = "saw.redhat.com/desired-sha256" +MANAGED = "saw.redhat.com/managed-by" +NAME = re.compile(r"[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\Z") +IMAGE = re.compile(r"[^\s@]+@sha256:[0-9a-f]{64}\Z") +QUANTITY = re.compile(r"[1-9][0-9]*(?:m|Ki|Mi|Gi|Ti)?\Z") + + +SAFE_FAILURE_REASONS = frozenset({ + "ProfileRejected", "InvalidPlan", "PlacementRejected", "IdentityConflict", + "InvalidIntent", "StaleIntent", "ReconcilerStateRejected", "GatewayMismatch", + "ProviderProfileMismatch", + "WorkspaceRejected", "ProviderRejected", "SandboxRejected", + "SandboxStateMismatch", "SandboxDrift", +}) + + +class ContractError(ValueError): + """An input is invalid or requires an unimplemented capability. + + ``safe_reason`` is an optional, fixed code for status reporting. Exception + text can contain private values, so callers must never publish it. + """ + + def __init__(self, message, *, safe_reason=None): + super().__init__(message) + self.safe_reason = (safe_reason if isinstance(safe_reason, str) + and safe_reason in SAFE_FAILURE_REASONS else None) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def digest(value): + return hashlib.sha256(canonical(value).encode()).hexdigest() + + +def request_id(owner, operation, value): + return str(uuid.uuid5(uuid.UUID(owner), operation + ":" + digest(value))) + + +def mapping(value, allowed, required, where): + if not isinstance(value, dict) or not all(isinstance(k, str) for k in value): + raise ContractError(f"{where} must be an object") + if set(value) - set(allowed) or set(required) - set(value): + raise ContractError(f"{where} has unknown or missing fields") + return value + + +def name(value, where, max_length=19): + if not isinstance(value, str) or len(value) > max_length or not NAME.fullmatch(value): + raise ContractError(f"{where} must be a DNS label of at most {max_length} characters") + return value + + +def _claim(binding, workspace, mount_path, volume_name): + mapping(binding, {"claimName", "uid", "readOnly"}, {"claimName", "uid", "readOnly"}, "PVC binding") + # Intentionally stricter than Kubernetes subdomains for this first adapter. + name(binding["claimName"], "claimName", 63) + if not isinstance(binding["uid"], str) or not binding["uid"]: + raise ContractError("PVC binding requires its observed UID") + if type(binding["readOnly"]) is not bool: + raise ContractError("PVC readOnly must be explicit boolean") + volume = {"name": volume_name, "persistent_volume_claim": { + "claim_name": binding["claimName"], "read_only": binding["readOnly"]}} + mount = {"name": volume_name, "mount_path": mount_path, "read_only": binding["readOnly"]} + reference = {"workspace": workspace, "name": binding["claimName"], "uid": binding["uid"]} + return volume, mount, reference + + +def _storage(binding, workspace): + mapping(binding, {"workspace", "mounts", "resources", "browser", "toolGate"}, {"workspace"}, "sandbox binding") + volume, mount, reference = _claim(binding["workspace"], workspace, "/sandbox/workspace", "saw-workspace") + if reference["name"].startswith(("saw-gate-", "saw-audit-")): + raise ContractError("reserved tool PVC cannot be used as a workspace") + if binding["workspace"]["readOnly"]: + raise ContractError("/sandbox/workspace must be writable") + volumes, mounts, refs = [volume], [mount], [reference] + if "toolGate" in binding: + gate = binding["toolGate"] + mapping(gate, {"trusted", "audit", "ownerLogin", "receiptKey", "modelSelector", "harness"}, {"trusted", "audit", "ownerLogin"}, "toolGate") + name(gate["ownerLogin"], "toolGate ownerLogin", 63) + for field, volume_name, path, readonly in [ + ("trusted", "saw-gate", "/sandbox/.saw/trusted", True), + ("audit", "saw-audit", "/sandbox/.saw/audit", False), + ]: + volume, mount, reference = _claim(gate[field], workspace, path, volume_name) + if not reference["name"].startswith(volume_name + "-") or mount["read_only"] != readonly: + raise ContractError(f"{field} requires the reserved claim prefix and mount mode") + mount["sub_path"] = "bundle" if readonly else "log" + volumes.append(volume) + mounts.append(mount) + refs.append(reference) + extra = binding.get("mounts", []) + if not isinstance(extra, list) or len(extra) > 16: + raise ContractError("mounts must be a list of at most 16 additional PVCs") + for i, item in enumerate(extra): + mapping(item, {"claimName", "uid", "readOnly", "mountPath"}, + {"claimName", "uid", "readOnly", "mountPath"}, "mount") + path = item["mountPath"] + # The driver reserves /sandbox itself. Mount the retained workspace below it. Additional data may + # only mount under /data, never over trusted/runtime/configuration trees. + if not isinstance(path, str) or not (path == "/data" or path.startswith("/data/")): + raise ContractError("additional mounts must be at /data or below it") + if str(PurePosixPath(path)) != path or ".." in PurePosixPath(path).parts or any(ord(c) < 32 for c in path): + raise ContractError("mountPath must be normalized") + for existing in mounts: + other = existing["mount_path"] + if path == other or path.startswith(other + "/") or other.startswith(path + "/"): + raise ContractError("overlapping mounts are not supported") + claim = {k: v for k, v in item.items() if k != "mountPath"} + volume, mount, reference = _claim(claim, workspace, path, f"data-{i}") + if reference["name"].startswith(("saw-gate-", "saw-audit-")): + raise ContractError("reserved tool PVC cannot be used as a data mount") + if any(r["name"] == reference["name"] for r in refs): + raise ContractError("one PVC cannot be attached through multiple bindings") + volumes.append(volume); mounts.append(mount); refs.append(reference) + resources = binding.get("resources", {}) + mapping(resources, {"requests", "limits"}, set(), "resources") + for group in resources.values(): + mapping(group, {"cpu", "memory", "ephemeral-storage"}, set(), "resource quantities") + if any(not isinstance(v, str) or not QUANTITY.fullmatch(v) for v in group.values()): + raise ContractError("resource quantities must be positive supported quantity strings") + return {"volumes": volumes, "containers": {"agent": { + "volume_mounts": mounts, "resources": resources}}}, refs + + +def compile_profile(profile_dir, config): + """Directory frontend for the same flat ConfigMap compiler used by the controller.""" + root = Path(profile_dir) + if not root.is_dir() or root.is_symlink(): + raise ContractError("profile path must be a real directory") + files = {} + for directory in sorted(root.iterdir()): + if not directory.is_dir() or directory.is_symlink(): + raise ContractError("profile must contain workspace directories only") + if "__" in directory.name: + raise ContractError("workspace directory cannot contain a flat-key separator") + for path in sorted(directory.iterdir()): + if path.is_symlink() or not path.is_file(): + raise ContractError("profile documents must be regular files, not symlinks") + try: + files[f"profiles__input__{directory.name}__{path.name}"] = path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + raise ContractError("cannot read profile document") from None + return compile_profile_files(files, config) + + +def _openclaw_startup(kind, info): + slot, _ = governed.requirements(kind, info.get("baseUrl")) + endpoint = info.get("baseUrl") or shared.NATIVE_BASE_URLS[kind] + provider_id = shared.openclaw_managed_provider_id(kind) + timeout = info.get("timeoutSeconds", 0) + return (Path(__file__).with_name("openclaw.sh").read_text() + .replace("@SAW_PROVIDER_KEY@", json.dumps(provider_id)) + .replace("@SAW_BASE_URL@", json.dumps(endpoint)) + .replace("@SAW_TIMEOUT@", f"timeoutSeconds: {timeout}, " if timeout else "") + .replace("@SAW_API@", "anthropic-messages" if info["protocol"] == "anthropic" else "openai-completions") + .replace("@SAW_KEY_ENV@", slot["env_vars"][0]) + .replace("@SAW_PROVIDER_ID@", provider_id)) + + +def _compatible_inference(workspace, sandbox, config): + """Resolve the shared BOM against explicit credential and public-input bindings.""" + provider = shared.select_inference_provider( + workspace.providers, sandbox.providers, + inference_ref=(sandbox.inference or {}).get("providerRef")) + if provider is None or provider.type not in {"openai", "anthropic", "nvidia", "custom-inference"}: + raise ContractError("agent requires a supported bound inference provider") + custom = provider.type == "custom-inference" + protocol = "openai" if custom else shared.PROVIDER_PROTOCOLS[provider.type] + endpoint = (config.get("customInferenceURL") if custom else + shared.NATIVE_BASE_URLS.get(provider.type) or shared.PROTOCOL_NATIVE_BASE_URLS[protocol]) + if custom: + try: + governed.custom_host(endpoint) + except (ValueError, TypeError) as exc: + raise ContractError("custom inference endpoint is not approved") from exc + inputs = config.get("modelInputs", {}) + credentials = {(b["secret"], b["key"]) for b in config.get("proxyBindings", [])} + def read_model(secret, key): + if secret != provider.credential_secret or (secret, key) in credentials: + raise ContractError("model input is outside the public inference binding") + if custom and key == "url": + return endpoint + if custom and key != "model": + raise ContractError("unapproved custom inference input") + return inputs.get(secret + "." + key, "") + try: + if sandbox.inference: + info = shared.resolve_inference(workspace, sandbox, read_model, lambda *_: None, + forbidden=credentials) + else: + info = {"providerRef": provider.name, + "protocol": shared.sandbox_protocol(sandbox, provider), + "baseUrl": shared.sandbox_endpoint(sandbox, provider), + "model": shared.sandbox_model(sandbox, provider), "timeoutSeconds": 0} + except shared.InstallerError as exc: + raise ContractError("invalid sandbox inference inputs") from exc + if (provider.name not in sandbox.providers or info["baseUrl"] not in ("", endpoint) + or info["protocol"] != protocol or info.get("models") + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,180}", info["model"])): + raise ContractError("inference inputs differ from the approved endpoint or supported protocol") + info["baseUrl"] = endpoint + return provider, info + + +def compile_profile_files(files, config): + """Compile stock SAW ConfigMap data with administrator-approved bindings. + + The shared VM/Kubernetes core owns document parsing, models and semantic + validation. This adapter adds only backend capabilities, attachment approval + inputs and the pinned Kubernetes driver envelope. No credentials are read. + """ + if not isinstance(files, dict) or any(not isinstance(k, str) or not isinstance(v, str) + for k, v in files.items()): + raise ContractError("profile data must be a mapping of flat file names to YAML text") + try: + documents, raw_harness, index = shared.split_bom_files(files) + bundles = harness.read_bundles(raw_harness, index) + profiles = shared.parse_profiles(documents, strict=True, document_check=_check_document_capabilities) + shared.validate_profiles(profiles, governed=True) + except shared.InstallerError: + # Parser errors may contain input snippets. The controller publishes + # status; do not accidentally turn that into a profile/secret echo. + raise ContractError("profile documents fail shared SAW schema or semantic validation") from None + mapping(config, {"ownerID", "namespace", "namespaceUID", "ownerSubject", "mode", + "workloadRuntimeClass", "workloadRuntimeClassUID", "workloadRuntimeClassDigest", "bindings", + "approvedCredentialFreeProviders", "gatewayVersion", "retiredSandboxes", "modelInputs", "proxyBindings", "customInferenceURL"}, + {"ownerID", "namespace", "namespaceUID", "ownerSubject", "mode", "bindings", + "approvedCredentialFreeProviders", "gatewayVersion"}, "administrator configuration") + try: + owner = str(uuid.UUID(config["ownerID"])) + if uuid.UUID(owner).int == 0: + raise ValueError() + except (ValueError, TypeError, AttributeError): + raise ContractError("ownerID must be a nonzero UUID") from None + name(config["namespace"], "namespace", 63) + for key in ["namespaceUID", "ownerSubject", "gatewayVersion"]: + if not isinstance(config[key], str) or not config[key].strip(): + raise ContractError(f"{key} must be a nonempty string") + mode = config["mode"] + runtime = config.get("workloadRuntimeClass", "") + runtime_ref = {"name": runtime, "uid": config.get("workloadRuntimeClassUID", "")} + if mode == "container": + if any(runtime_ref.values()) or "workloadRuntimeClassDigest" in config: + raise ContractError("container mode must not select a RuntimeClass") + elif mode == "kata": + name(runtime, "workload RuntimeClass", 63) + try: + uid = uuid.UUID(runtime_ref["uid"]) + if str(uid) != runtime_ref["uid"] or not uid.int: + raise ValueError() + except (ValueError, AttributeError, TypeError): + raise ContractError("kata requires a reviewed RuntimeClass UID") from None + sha = config.get("workloadRuntimeClassDigest") + if not isinstance(sha, str) or not re.fullmatch("[a-f0-9]{64}", sha): + raise ContractError("kata requires a reviewed RuntimeClass digest") + runtime_ref["sha256"] = sha + else: + raise ContractError("this adapter only accepts container or kata") + approved = config["approvedCredentialFreeProviders"] + if not isinstance(approved, list) or any(not isinstance(p, str) for p in approved): + raise ContractError("approvedCredentialFreeProviders must be a list") + bindings = config["bindings"] + if not isinstance(bindings, dict): + raise ContractError("bindings must be an object") + workspaces, all_refs, used_bindings = [], [], set() + for _, workspace in shared.enabled_workspaces(profiles): + ws_name = name(workspace.name, "workspace") + providers = [] + for provider in workspace.providers: + if not provider.enabled: + continue + name(provider.name, "provider") + name(provider.type, "provider type") + # All semantic checks (references, credentialFree, governed names) + # live in the common core. The trusted proxy owns private credentials and refresh. + entry = {"metadata": {"name": provider.name, "labels": { + MANAGED: "kubernetes-runtime", OWNER: owner}}, "type": provider.type} + if not config.get("proxyBindings") and (provider.refresh_strategy or provider.credentials_spec or provider.credential_secret_key != "api_key"): + raise ContractError("provider requires an unported credential/refresh capability or approval") + if provider.credential_free: + if provider.type == "custom-inference": + raise ContractError("custom inference requires a bound credential") + if provider.type not in approved: + raise ContractError("credential-free provider requires administrator approval") + elif config.get("proxyBindings"): + try: + entry["credentials"] = governed.checked_marker(provider, config["proxyBindings"], config.get("customInferenceURL")) + except ValueError as exc: + raise ContractError(str(exc)) from exc + else: + raise ContractError("provider requires an unported credential/refresh capability or approval") + providers.append(entry) + sandboxes = [] + for sandbox in workspace.sandboxes: + if not sandbox.enabled: + continue + name(sandbox.name, "sandbox") + if sandbox.gpu_enabled: + raise ContractError("container workspaces do not support GPU device requests") + command = ["sh", "-c", "sleep infinity"] + if sandbox.env or sandbox.model or sandbox.agent != "openclaw": + raise ContractError("harness/UI/inference/env capability is not yet ported to Kubernetes") + if sandbox.ui and (sandbox.type not in ("openclaw","nemoclaw") or sandbox.ui != {"route": True} + or not bindings.get(f"{ws_name}/{sandbox.name}", {}).get("browser")): + raise ContractError("UI route capability requires the matching approved OpenClaw browser binding") + if sandbox.type in ("openclaw", "nemoclaw"): + inference_provider, info = _compatible_inference(workspace, sandbox, config) + if not any(p["metadata"]["name"] == inference_provider.name and p.get("credentials") + for p in providers): + raise ContractError("agent requires an explicitly bound inference credential") + kind = inference_provider.type + script = _openclaw_startup(kind, info) + browser = bindings.get(f"{ws_name}/{sandbox.name}", {}).get("browser") + extra = "" + if browser is not None: + if "toolGate" not in bindings[f"{ws_name}/{sandbox.name}"]: + raise ContractError("browser requires the read-only tool gate for its relay proof") + mapping(browser, {"origin", "ownerLogin"}, {"origin", "ownerLogin"}, "browser") + origin, login = browser["origin"], browser["ownerLogin"] + if (not isinstance(origin, str) or not re.fullmatch(r"https://[a-z0-9.-]+", origin) + or not isinstance(login, str) or not NAME.fullmatch(login)): + raise ContractError("browser requires an explicit HTTPS origin and owner login") + extra = ("const proof = fs.readFileSync('/sandbox/.saw/trusted/relay-proof','ascii');\n" + "if (!/^[a-f0-9]{64}$/.test(proof)) throw new Error('invalid browser relay proof');\n" + "config.gateway.trustedProxies = ['127.0.0.1/32','::1/128'];\n" + "config.gateway.controlUi = {allowedOrigins:[" + json.dumps(origin) + "]};\n" + "config.gateway.auth = {mode:'trusted-proxy',password:config.gateway.auth.token," + "trustedProxy:{userHeader:'x-saw-relay-identity',allowUsers:[" + json.dumps(config["ownerSubject"]) + " + '.' + proof]," + "allowLoopback:true,deviceAutoApprove:{enabled:true,scopes:['operator.read','operator.write']}}};\n") + script = script.replace("@SAW_BROWSER_CONFIG@\n", extra) + command = ["sh", "-ec", script, "saw-openclaw", info["model"]] + elif sandbox.type != "generic" or sandbox.inference: + raise ContractError("agent or inference capability is not yet ported to Kubernetes") + if not IMAGE.fullmatch(sandbox.image): + raise ContractError("sandbox image must be pinned by sha256 digest") + references = sandbox.providers + if len(set(references)) != len(references): + raise ContractError("sandbox provider references must be unique") + key = f"{ws_name}/{sandbox.name}" + if key not in bindings: + raise ContractError(f"{key} requires an administrator-approved retained workspace PVC") + used_bindings.add(key) + if sandbox.type not in ("openclaw", "nemoclaw") and "browser" in bindings[key]: + raise ContractError("browser settings require an OpenClaw sandbox") + driver, refs = _storage(bindings[key], ws_name) + all_refs.extend(refs) + if "toolGate" in bindings[key]: + if sandbox.type not in ("openclaw", "nemoclaw"): + raise ContractError("toolGate requires an OpenClaw sandbox") + login = bindings[key]["toolGate"]["ownerLogin"] + if browser and login != browser["ownerLogin"]: + raise ContractError("browser and tool gate owners must match") + identity = shared.tool_gate_identity(login, config["ownerSubject"], ws_name, sandbox.name) + assets = Path(shared.__file__).parent + receipt_key = bindings[key]["toolGate"].get("receiptKey") + try: + _, _, gate_digest, _ = shared.tool_gate_bundle(assets, identity, receipt_key) + except shared.InstallerError as exc: + raise ContractError(str(exc)) from exc + script = Path(__file__).with_name("tool_gate_preflight.js").read_text() + if bindings[key]["toolGate"].get("modelSelector") is True: + if not receipt_key: + raise ContractError("mutable model requires a trusted gate") + # The shared model resolver validates the original choice. + # The command is stable across an owner's later selections. + command[-1] = "model-from-gate" + command[2] = command[2].replace("const model = process.argv[2];", """ +const selection = JSON.parse(fs.readFileSync('/sandbox/.saw/trusted/model.json','utf8')); +if (!Number.isSafeInteger(selection.generation) || selection.generation < 1 || + typeof selection.model !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(selection.model)) + throw new Error('invalid published model'); +const model = selection.model; +""") + if kind in {"custom-inference", "nvidia"}: + command[2] = command[2].replace("[A-Za-z0-9._-]{0,99}", r"[A-Za-z0-9._\/-]{0,99}") + script = script.replace("for (const path of treeFiles(root).sort()) {", + "for (const path of treeFiles(root).sort()) {\n if (path === root + '/model.json') continue;") + if browser: + script = script.replace("for (const path of treeFiles(root).sort()) {", + "for (const path of treeFiles(root).sort()) {\n if (path === root + '/relay-proof') continue;") + if receipt_key: + # Keep the VM-style file preflight for previously published plans. + script = script[:script.index("if (!fs.lstatSync(audit)")] + """ +config.plugins = {allow:['saw-tool-gate'],load:{paths:[root+'/plugin']}, + entries:{'saw-tool-gate':{enabled:true}}}; +config.tools = {allow:['read','write','exec']}; +""" + command[2] = command[2].replace("exec openclaw gateway", """openclaw plugins list --json > "$TMPDIR/saw-plugins.json" +node -e 'const p=JSON.parse(require("fs").readFileSync(process.argv[1])).plugins.find(x=>x.id==="saw-tool-gate");if(!p?.enabled||p.status!=="loaded"||p.rootDir!=="/sandbox/.saw/trusted/plugin")process.exit(1)' "$TMPDIR/saw-plugins.json" +exec openclaw gateway""") + if browser: + command[2] = command[2].replace("scopes:['operator.read','operator.write']", + "scopes:['operator.read','operator.write','operator.approvals']") + command[2] = command[2].replace("const target = process.env.OPENCLAW_CONFIG_PATH;", + script.replace("@SAW_GATE_DIGEST@", gate_digest) + "\nconst target = process.env.OPENCLAW_CONFIG_PATH;") + gate = bindings[key].get("toolGate", {}) + try: + expected_harness = harness.binding_for(sandbox, bundles) + if expected_harness != gate.get("harness"): + raise ContractError("sandbox harness differs from its enrolled binding") + if expected_harness: + if not gate.get("receiptKey") or sandbox.type not in ("openclaw", "nemoclaw"): + raise ContractError("harness requires an enrolled agent and signed tool gate") + provider_types = {p.type for p in workspace.providers if p.enabled and p.name in sandbox.providers} + catalog = {t: governed.requirements(t)[1] for t in provider_types if t in governed.SUPPORTED} + harness.configure(command, driver, bundles[expected_harness["name"]], workspace, sandbox, catalog) + except shared.InstallerError as exc: + raise ContractError(str(exc)) from exc + spec = {"template": {"image": sandbox.image, "driver_config": {"kubernetes": driver}}, + "providers": sorted(references), "command": command, "tty": False} + if sandbox.type in ("openclaw", "nemoclaw"): + # The boundary binary runs before the main command and inherits + # image TMPDIR. OpenShift assigns a distinct UID per namespace. + spec["template"]["environment"] = {"TMPDIR": "/tmp"} + sandboxes.append({"name": sandbox.name, "spec": spec, "annotations": {REVISION: digest(spec)}, + "labels": {MANAGED: "kubernetes-runtime", OWNER: owner}}) + workspaces.append({"name": ws_name, "providers": sorted(providers, key=lambda p: p["metadata"]["name"]), + "sandboxes": sorted(sandboxes, key=lambda s: s["name"])}) + if not workspaces or not any(w["sandboxes"] for w in workspaces): + raise ContractError("profile must contain at least one enabled generic sandbox") + if set(bindings) != used_bindings: + raise ContractError("unused administrator bindings must be removed") + claims = [r["name"] for r in all_refs] + if len(claims) != len(set(claims)): + raise ContractError("sharing retained claims between sandboxes is not supported") + plan = {"schemaVersion": 1, "sourceRevision": SOURCE_REVISION, "ownerID": owner, + "namespace": config["namespace"], "namespaceUID": config["namespaceUID"], + "ownerSubject": config["ownerSubject"], "mode": mode, "gatewayVersion": config["gatewayVersion"], + "requestedPlacement": {"supervisor": "container", "workload": mode}, + "runtimeClass": runtime_ref, + "claims": sorted(all_refs, key=lambda r: (r["workspace"], r["name"])), + "workspaces": sorted(workspaces, key=lambda w: w["name"])} + if any(p["type"] == "custom-inference" for w in workspaces for p in w["providers"]): + plan["customInferenceURL"] = config["customInferenceURL"] + retired = config.get("retiredSandboxes", {}) + if not isinstance(retired, dict): + raise ContractError("retiredSandboxes must map workspace/sandbox names to retained IDs") + for target, identifier in retired.items(): + if (not isinstance(target, str) or target.count("/") != 1 or target in used_bindings + or not all(NAME.fullmatch(part) for part in target.split("/"))): + raise ContractError("retired sandbox must be a distinct, named former workload") + try: + if str(uuid.UUID(identifier)) != identifier: + raise ValueError() + except (ValueError, TypeError, AttributeError): + raise ContractError("retired sandbox requires its recorded UUID") from None + if retired: + plan["retiredSandboxes"] = dict(retired) + return {**plan, "revision": digest(plan)} + + +def _check_document_capabilities(document, kind): + # These fields belong to SAW's common language but are consumed by the VM + # chart, not its normalized installer models. Reject before normalization + # so the Kubernetes path cannot silently drop UI/harness requirements. + if kind == "Sandboxes": + for entry in document.get("spec", {}).get("sandboxes", []): + if entry.get("enabled", True) and entry.get("harnessRequired") and not entry.get("harnessRef"): + raise ContractError("required harness capability has no harnessRef") diff --git a/kubernetes-runtime/saw_kubernetes/controller.py b/kubernetes-runtime/saw_kubernetes/controller.py new file mode 100644 index 000000000..de1a8b5df --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/controller.py @@ -0,0 +1,455 @@ +"""Single-writer ConfigMap frontend; no cluster connection at import time. + +The host application must supply the profile compiler, core pass and a real +boundary verifier. No default verifier or implicit resource deletion is provided. +Use a retained private state directory distinct from the core reconciler's state. +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +import json +from pathlib import Path +import re +import ssl +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urlsplit +from urllib.request import HTTPRedirectHandler, HTTPSHandler, Request, build_opener +import uuid + +from .compiler import ContractError, OWNER, canonical, digest +from .reconcile import MissingSandboxReceipt, RpcError, StateStore + +MAX_BODY = 2 * 1024 * 1024 +MAX_DATA = 900 * 1024 +MANAGED = "saw.redhat.com/managed-by" +CONTROLLER = "kubernetes-runtime-controller" +# Kubernetes DNS subdomains require every dot-separated label, not only the +# entire name, to start and end with an alphanumeric character. +NAME = re.compile(r"[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*\Z") +KEY = re.compile(r"[A-Za-z0-9_.-]+\Z") +CORE_PHASES = {"Pending", "Retry", "GenericReady", "Stopped"} + + +class ApiError(Exception): + """Only the numeric HTTP code crosses the client boundary.""" + + def __init__(self, code=0): + self.code = code if type(code) is int and 0 <= code <= 599 else 0 + super().__init__(f"Kubernetes API failure ({self.code})") + + +class InputError(Exception): + def __init__(self, reason): + self.reason = reason + super().__init__(reason) + + +def valid_lifecycle_intent(intent): + """Envelope shared by the controller and the local intent generator.""" + fields = {"schemaVersion", "generation", "desiredState", "profileRef", "bindings"} + if not isinstance(intent, dict) or set(intent) - {"restartGeneration"} != fields: + return False + generation = intent["generation"] + restart = intent.get("restartGeneration", 0) + return (type(intent["schemaVersion"]) is int and intent["schemaVersion"] == 1 + and type(generation) is int and generation >= 1 and type(restart) is int + and 0 <= restart <= generation and intent["desiredState"] in ("Running", "Stopped")) + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None # Never forward the projected bearer token to another URL. + + +@dataclass(frozen=True) +class Scope: + namespace: str + profile_name: str + bindings_name: str + status_name: str + owner_id: str + + def __post_init__(self): + for value in [self.namespace, self.profile_name, self.bindings_name, self.status_name]: + if not isinstance(value, str) or len(value) > 253 or not NAME.fullmatch(value) or ".." in value: + raise ContractError("controller scope requires explicit Kubernetes names") + if len(self.namespace) > 63 or "." in self.namespace: + raise ContractError("controller namespace must be a DNS label") + if len({self.profile_name, self.bindings_name, self.status_name}) != 3: + raise ContractError("profile, bindings and status ConfigMaps must be distinct") + try: + if str(uuid.UUID(self.owner_id)) != self.owner_id or uuid.UUID(self.owner_id).int == 0: + raise ValueError() + except (ValueError, TypeError, AttributeError): + raise ContractError("controller owner must be a canonical nonzero UUID") from None + + +class KubernetesClient: + """Verified HTTPS with a rotating projected token and no redirects. + + Callers constrain resource paths; Kubernetes RBAC constrains authorization. + API responses/error bodies and bearer tokens are never included in errors. + """ + + def __init__(self, server, ca_file, token_file, *, timeout=20, opener=None): + try: + url = urlsplit(server) + port = url.port + except (TypeError, ValueError): + raise ContractError("Kubernetes API server must be an HTTPS authority") from None + if (url.scheme != "https" or not url.hostname or url.username or url.password + or url.path not in {"", "/"} or url.query or url.fragment or port == 0 + or any(c.isspace() for c in server)): + raise ContractError("Kubernetes API server must be an HTTPS authority") + if type(timeout) not in {int, float} or not 0 < timeout <= 45: + raise ContractError("API timeout must be between zero and 45 seconds") + try: + self.context = ssl.create_default_context(cafile=str(ca_file)) + except (OSError, ssl.SSLError): + raise ContractError("a valid Kubernetes CA file is required") from None + self.server = server.rstrip("/") + self.token_file, self.timeout = Path(token_file), timeout + if opener is None: + verified = build_opener(HTTPSHandler(context=self.context), _NoRedirect()) + opener = lambda request, **kwargs: verified.open(request, timeout=kwargs["timeout"]) + self.opener = opener + + def _open(self, method, path, body=None, *, timeout=None): + try: + token = self.token_file.read_text().strip() + if not token or any(c.isspace() for c in token): + raise ValueError() + except (OSError, ValueError): + raise ApiError() from None + headers = {"Authorization": "Bearer " + token, "Accept": "application/json"} + raw = None if body is None else canonical(body).encode() + if raw is not None: + if len(raw) > MAX_BODY: + raise ApiError() + headers["Content-Type"] = "application/json" + request = Request(self.server + path, data=raw, headers=headers, method=method) + try: + return self.opener(request, context=self.context, timeout=timeout or self.timeout) + except HTTPError as exc: + code = exc.code + exc.close() + raise ApiError(code) from None + except (OSError, URLError, ValueError): + raise ApiError() from None + + @staticmethod + def _json(response): + try: + raw = response.read(MAX_BODY + 1) + if len(raw) > MAX_BODY: + raise ValueError() + value = json.loads(raw) + if not isinstance(value, dict): + raise ValueError() + return value + except (OSError, ValueError, UnicodeError, RecursionError): + raise ApiError() from None + + def request(self, method, path, body=None): + with self._open(method, path, body) as response: + return self._json(response) + + +class ConfigMapClient(KubernetesClient): + """Named ConfigMap reads and status-only writes over the shared HTTP client.""" + def __init__(self, server, scope, ca_file, token_file, *, timeout=20, opener=None): + super().__init__(server, ca_file, token_file, timeout=timeout, opener=opener) + self.scope = scope + + def _path(self, name): + if name not in {self.scope.profile_name, self.scope.bindings_name, self.scope.status_name}: + raise ContractError("ConfigMap access is outside the controller allowlist") + return f"/api/v1/namespaces/{self.scope.namespace}/configmaps/{name}" + + def get(self, name): + try: + with self._open("GET", self._path(name)) as response: + return self._json(response) + except ApiError as exc: + if exc.code == 404: + return None + raise + + def write_status(self, body, *, resource_version=None): + name = self.scope.status_name + metadata = body.get("metadata", {}) + if metadata.get("name") != name or metadata.get("namespace") != self.scope.namespace: + raise ContractError("only the controller status ConfigMap may be written") + if resource_version is None: + path, method = self._path(name).rsplit("/", 1)[0], "POST" + else: + if metadata.get("resourceVersion") != resource_version: + raise ContractError("status update must preserve its observed resourceVersion") + path, method = self._path(name), "PUT" + with self._open(method, path, body) as response: + return self._json(response) + + def watch(self, name, resource_version, *, timeout_seconds=20): + self._path(name) + if name != self.scope.bindings_name or type(timeout_seconds) is not int or not 1 <= timeout_seconds <= 30: + raise ContractError("only a bounded admin-intent watch is permitted") + query = urlencode({"fieldSelector": f"metadata.name={name}", "watch": "true", + "resourceVersion": resource_version, "timeoutSeconds": timeout_seconds, + "allowWatchBookmarks": "true"}) + path = self._path(name).rsplit("/", 1)[0] + "?" + query + try: + with self._open("GET", path, timeout=timeout_seconds + 5) as response: + deadline = time.monotonic() + timeout_seconds + for _ in range(100): + if time.monotonic() >= deadline: + return + raw = response.readline(MAX_BODY + 1) + if not raw: + return + if len(raw) > MAX_BODY: + raise ApiError() + event = json.loads(raw) + if not isinstance(event, dict): + raise ApiError() + if event.get("type") == "ERROR": + error = event.get("object") + if not isinstance(error, dict): + raise ApiError() + raise ApiError(error.get("code", 0)) + yield event + except (OSError, ValueError, UnicodeError, RecursionError): + raise ApiError() from None + + +class Controller: + """One process/replica with retained state; not a distributed lease or HA. + + `compile_files(flat_data, bindings)` and `core_pass(plan, generation=..., + desired_state=...)` are injected. `verify_boundary(plan)` is mandatory and + must actually validate the installation's trust and enforcement prerequisites. + A separate state subdirectory makes the lifetime lock compose with the core's + per-pass lock. Missing/deleted inputs only block; they never mean purge/stop. + """ + + def __init__(self, api, scope, state_directory, compile_files, core_pass, verify_boundary): + if not all(callable(f) for f in [compile_files, core_pass, verify_boundary]): + raise ContractError("compiler, core pass and real boundary verifier are required") + self.api, self.scope = api, scope + self.compile_files, self.core_pass, self.verify_boundary = compile_files, core_pass, verify_boundary + self.store = StateStore(Path(state_directory) / "controller-frontend") + self.watch_version = "" + + def _metadata(self, obj, name): + if obj is None: + raise InputError("MissingInputs") + meta = obj.get("metadata", {}) + if (obj.get("kind") != "ConfigMap" or meta.get("name") != name + or meta.get("namespace") != self.scope.namespace + or not isinstance(meta.get("uid"), str) or not meta["uid"] + or not isinstance(meta.get("resourceVersion"), str) or not meta["resourceVersion"]): + raise InputError("InvalidInputs") + if meta.get("deletionTimestamp"): + raise InputError("DeletingInputs") + return meta + + @staticmethod + def _data(obj): + data = obj.get("data", {}) + if (obj.get("binaryData") or not isinstance(data, dict) or not data + or any(not isinstance(k, str) or not KEY.fullmatch(k) or k in {".", ".."} + or not isinstance(v, str) for k, v in data.items()) + or len(canonical(data).encode()) > MAX_DATA): + raise InputError("InvalidInputs") + return data + + def _snapshot(self): + bindings = self.api.get(self.scope.bindings_name) + bm = self._metadata(bindings, self.scope.bindings_name) + self.watch_version = bm["resourceVersion"] + if bm.get("labels", {}).get(OWNER) != self.scope.owner_id: + raise InputError("IdentityMismatch") + data = self._data(bindings) + if set(data) != {"intent.json"}: + raise InputError("InvalidInputs") + try: + intent = json.loads(data["intent.json"]) + except ValueError: + raise InputError("InvalidInputs") from None + if (not valid_lifecycle_intent(intent) + or not isinstance(intent["bindings"], dict) + or intent["bindings"].get("ownerID") != self.scope.owner_id + or intent["bindings"].get("namespace") != self.scope.namespace): + raise InputError("InvalidInputs") + reference = intent["profileRef"] + if (not isinstance(reference, dict) or set(reference) not in ({"name", "uid", "sha256"}, {"name", "uid", "sha256", "planSHA256"}) + or reference["name"] != self.scope.profile_name): + raise InputError("InvalidInputs") + profile = self.api.get(self.scope.profile_name) + pm = self._metadata(profile, self.scope.profile_name) + files = self._data(profile) + if reference["uid"] != pm["uid"] or reference["sha256"] != digest(files): + raise InputError("InputRevisionMismatch") + # K8s resourceVersions are opaque; equality, not numeric ordering. + for name, first in [(self.scope.bindings_name, bindings), (self.scope.profile_name, profile)]: + again = self.api.get(name) + self._metadata(again, name) + if again != first: + raise InputError("InputsChangedDuringRead") + return intent, files, {"bindings": bm["uid"], "profile": pm["uid"]}, digest({"intent": intent, "files": files}) + + def _status_object(self): + existing = self.api.get(self.scope.status_name) + if existing is not None: + meta = self._metadata(existing, self.scope.status_name) + labels = meta.get("labels", {}) + if labels.get(OWNER) != self.scope.owner_id or labels.get(MANAGED) != CONTROLLER: + raise InputError("StatusOwnershipMismatch") + return existing + + def _publish(self, status): + for _ in range(3): + existing = self._status_object() + resource_version = None + if existing is not None: + resource_version = existing["metadata"]["resourceVersion"] + if existing.get("data") == {"status.json": canonical(status)}: + return + body = {"apiVersion": "v1", "kind": "ConfigMap", "metadata": { + "name": self.scope.status_name, "namespace": self.scope.namespace, + "labels": {OWNER: self.scope.owner_id, MANAGED: CONTROLLER}}, + "data": {"status.json": canonical(status)}} + if resource_version: + body["metadata"] = copy.deepcopy(existing["metadata"]) + try: + self.api.write_status(body, resource_version=resource_version) + return + except ApiError as exc: + if exc.code != 409: + raise + raise InputError("StatusConflict") + + def _once(self): + state = self.store.load() + identity = vars(self.scope) + if state and state.get("scope") != identity: + raise ContractError("retained controller scope differs; explicit recovery is required") + state["scope"] = identity + status = {"schemaVersion": 1, "phase": "Blocked", "reason": "InvalidInputs"} + retry = 5 + try: + self._status_object() # Refuse known foreign status ownership before core mutations. + intent, files, uids, revision = self._snapshot() + status.update(observedGeneration=intent["generation"], inputRevision=revision) + previous = state.get("intent", {}) + if state.get("inputUIDs", uids) != uids: + raise InputError("InputIdentityReplaced") + if (previous.get("generation", 0) > intent["generation"] + or previous.get("generation") == intent["generation"] and previous.get("revision") != revision): + raise InputError("StaleOrConflictingGeneration") + try: + plan = self.compile_files(copy.deepcopy(files), copy.deepcopy(intent["bindings"])) + except ContractError: + raise InputError("ProfileRejected") from None + if "planSHA256" in intent["profileRef"] and plan["revision"] != intent["profileRef"]["planSHA256"]: + raise InputError("CompiledPlanMismatch") + if intent["desiredState"] == "Running": + try: + self.verify_boundary(plan) + except Exception: + raise InputError("BoundaryRejected") from None + # Stopped delegates to the core's identity-checked shutdown path. + # Renewed network/attachment approval must not prevent stopping an + # already recorded sandbox. Input fencing above remains mandatory. + state.update(inputUIDs=uids, intent={"generation": intent["generation"], "revision": revision}) + self.store.save(state) # Fence identities and intent before any core mutation. + restart = intent.get("restartGeneration",0) + # Reserve two monotonically ordered core passes per owner generation + # for registrations opted into model rollover. Owner status keeps + # the externally visible generation and revision. + core_generation = intent["generation"] * (2 if "restartGeneration" in intent else 1) + applied = state.get("restartGeneration",0) + if restart < applied: + raise InputError("StaleOrConflictingGeneration") + if restart > applied: + # A confirmed Stopped phase is the durable barrier. Persist it + # before starting so retries cannot repeatedly stop a fresh Pod. + result = self.core_pass(plan,generation=core_generation - 1,desired_state="Stopped") + if result.get("phase") == "Stopped": + state["restartGeneration"] = restart + self.store.save(state) + if intent["desiredState"] == "Running": + result = {"phase":"Pending","retry_after_seconds":1} + else: + result = self.core_pass(plan, generation=core_generation, desired_state=intent["desiredState"]) + if not isinstance(result, dict) or result.get("phase") not in CORE_PHASES: + raise InputError("InvalidCoreStatus") + status.update(phase=result["phase"], reason="CoreObserved") + delay = result.get("retry_after_seconds", 5) + retry = min(max(delay, 1), 30) if type(delay) is int else 5 + if result["phase"] in {"GenericReady", "Stopped"}: + retry = 30 # Full input/ownership resync, even without watch events. + state["apiFailures"] = 0 + self.store.save(state) + except InputError as exc: + status["reason"] = exc.reason + except MissingSandboxReceipt: + status["reason"] = "StartToRecoverSandboxReceipt" + except ContractError as exc: + status["reason"] = exc.safe_reason or "CoreRejected" + except RpcError as exc: + if exc.code in {"UNAVAILABLE", "DEADLINE_EXCEEDED", "ABORTED", "RESOURCE_EXHAUSTED"}: + status.update(phase="Retry", reason="CoreUnavailable") + else: + status.update(reason="CoreRpcRejected", rpcCode=exc.code) + except ApiError: + failures = min(state.get("apiFailures", 0) + 1, 5) + state["apiFailures"] = failures + self.store.save(state) + retry = min(2 ** failures, 30) + status.update(phase="Retry", reason="ApiUnavailable") + except Exception: + # A caller's compiler/core exception may contain credentials. Never + # serialize str(exc), traceback, requests or returned RPC details. + status["reason"] = "CoreFailure" + status["retryAfterSeconds"] = retry + try: + self._publish(status) + except (ApiError, InputError): + # Desired state is never rewritten to recover a status conflict. + status["statusPublished"] = False + return status + + def reconcile_once(self): + with self.store.locked(): + return self._once() + + def run(self, *, stop=lambda: False, max_cycles=None, sleep=time.sleep, monotonic=time.monotonic): + """Lifetime lock, bounded watch, relist/resync after every event or 410. + + The injected watch must honor its timeout; the HTTPS client above does. + Only intent is watched; profile-only edits are caught by <=30s resync. + """ + cycles = 0 + with self.store.locked(): + while not stop() and (max_cycles is None or cycles < max_cycles): + status = self._once() + cycles += 1 + if stop() or max_cycles is not None and cycles >= max_cycles: + return status + delay = status["retryAfterSeconds"] + start = monotonic() + changed = False + try: + for event in self.api.watch(self.scope.bindings_name, self.watch_version, timeout_seconds=delay): + if event.get("type") in {"ADDED", "MODIFIED", "DELETED"}: + changed = True + break + except ApiError: + self.watch_version = "" # 410/transport loss always causes a fresh named GET. + elapsed = monotonic() - start + if not changed and elapsed < delay: + sleep(delay - elapsed) # EOF/errors honor the bounded retry delay. + return None diff --git a/kubernetes-runtime/saw_kubernetes/core.py b/kubernetes-runtime/saw_kubernetes/core.py new file mode 100644 index 000000000..65c6b5d74 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/core.py @@ -0,0 +1,40 @@ +"""Load the canonical shared SAW engine, never a module supplied by a profile. + +The repository and controller image retain the same layout. The VM installer +embeds these source bytes in its signed bundle; both loaders use a content-keyed +module name so a process importing both frontends gets identical model classes. +""" + +import hashlib +from pathlib import Path +import sys +import types + + +def load_shared(name): + if name not in {"core", "admin"}: + raise ValueError("unknown shared SAW source") + source_path = (Path(__file__).resolve().parents[2] / "charts" / "openshell-saw" + / "files" / "installer" / ("saw_"+name+".py")) + if source_path.is_symlink(): + raise RuntimeError("shared SAW core must be the trusted repository source file") + source = source_path.read_bytes() + module_name = "_saw_shared_"+name+"_" + hashlib.sha256(source).hexdigest() + if module_name not in sys.modules: + module = types.ModuleType(module_name) + module.__file__ = str(source_path) + sys.modules[module_name] = module + try: + exec(compile(source, str(source_path), "exec"), module.__dict__) + except BaseException: + sys.modules.pop(module_name, None) + raise + return sys.modules[module_name] + + +def load_core(): + return load_shared("core") + + +shared = load_core() +admin = load_shared("admin") diff --git a/kubernetes-runtime/saw_kubernetes/gate.py b/kubernetes-runtime/saw_kubernetes/gate.py new file mode 100644 index 000000000..b72efb399 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/gate.py @@ -0,0 +1,48 @@ +"""Render the VM tool gate as an immutable Kubernetes ConfigMap and its projection.""" + +import argparse +import json +import sys +from pathlib import Path + +from .compiler import name +from .core import shared + + +def projection(namespace, owner, subject, workspace, sandbox): + name(namespace, "namespace", 63) + name(workspace, "workspace") + name(sandbox, "sandbox") + identity = shared.tool_gate_identity(owner, subject, workspace, sandbox) + assets = Path(__file__).resolve().parents[2] / "charts/openshell-saw/files/installer" + tree, _, digest, _ = shared.tool_gate_bundle(assets, identity) + cm_name = "saw-tool-gate-" + digest.removeprefix("sha256:")[:32] + # The volume projects the six regular files at the same paths as the VM. + items = [{"key": path.replace("/", "__"), "path": path} for path in sorted(tree)] + data = {item["key"]: tree[item["path"]][0].decode() for item in items} + configmap = { + "apiVersion": "v1", "kind": "ConfigMap", "immutable": True, + "metadata": {"name": cm_name, "namespace": namespace, + "annotations": {"saw.redhat.com/tool-gate-sha256": digest}}, + "data": data, + } + return configmap, {"name": cm_name, "defaultMode": 0o444, "items": items} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for key in ("namespace", "owner", "subject", "workspace", "sandbox"): + parser.add_argument("--" + key, required=True) + parser.add_argument("--output", choices=("configmap", "volume-source"), default="configmap") + args = vars(parser.parse_args()) + output = args.pop("output") + try: + cm, source = projection(**args) + except (ValueError, shared.InstallerError) as exc: + parser.exit(2, str(exc) + "\n") + json.dump(cm if output == "configmap" else source, sys.stdout, indent=2) + print() + + +if __name__ == "__main__": + main() diff --git a/kubernetes-runtime/saw_kubernetes/gateway_ca.py b/kubernetes-runtime/saw_kubernetes/gateway_ca.py new file mode 100644 index 000000000..690387ff6 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/gateway_ca.py @@ -0,0 +1,43 @@ +"""Read the owner-bound public gateway CA from its workspace namespace.""" +import re +import ssl + + +NAME = re.compile(r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?\Z") +OWNER = "saw.redhat.com/runtime-owner" +INVALID = "public gateway CA is missing, foreign or malformed" + + +def configmap_path(namespace, gateway): + if any(not isinstance(value, str) or not NAME.fullmatch(value) + for value in (namespace, gateway)): + raise ValueError(INVALID) + return f"/api/v1/namespaces/{namespace}/configmaps/{gateway}-gateway-ca" + + +def read_public_ca(namespace, gateway, owner_id, get): + """Read one immutable public CA ConfigMap. Never accept an unowned map.""" + obj = get(configmap_path(namespace, gateway)) + if not isinstance(obj, dict): + raise ValueError(INVALID) + metadata, data = obj.get("metadata"), obj.get("data") + if not isinstance(metadata, dict) or not isinstance(data, dict): + raise ValueError(INVALID) + labels = metadata.get("labels") + ca = data.get("ca.crt") + if (not isinstance(labels, dict) or obj.get("apiVersion") != "v1" + or obj.get("kind") != "ConfigMap" + or metadata.get("name") != gateway + "-gateway-ca" + or metadata.get("namespace") != namespace + or labels.get(OWNER) != owner_id + or metadata.get("deletionTimestamp") or metadata.get("ownerReferences") + or obj.get("immutable") is not True + or set(data) != {"ca.crt"} + or not isinstance(ca, str) + or not re.fullmatch(r"-----BEGIN CERTIFICATE-----\s+[A-Za-z0-9+/=\s]+-----END CERTIFICATE-----\s*", ca)): + raise ValueError(INVALID) + try: + ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=ca) + except (ssl.SSLError, ValueError): + raise ValueError(INVALID) from None + return ca diff --git a/kubernetes-runtime/saw_kubernetes/harness.py b/kubernetes-runtime/saw_kubernetes/harness.py new file mode 100644 index 000000000..eadc2a73d --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/harness.py @@ -0,0 +1,62 @@ +"""Kubernetes delivery of the shared VM inline harness format.""" +import base64 +import json +import re +from pathlib import Path + +import yaml + +from .core import shared + + +def read_bundles(raw, index): + bundles = shared.parse_harness_files(raw) + if bundles and not isinstance(index.get("bundles"), dict): + raise shared.InstallerError("inline harness requires the Helm digest index") + shared.check_harness_index(bundles, index) + return bundles + + +def bundle_tree(bundle): + return {rel: (base64.b64decode(data, validate=True), False) for rel, data in bundle.files.items()} + + +def binding_for(sandbox, bundles): + if not sandbox.harness_ref: + return None + if sandbox.harness_ref.get("image"): + raise shared.InstallerError("OCI harness verification is unavailable in the Kubernetes runtime") + bundle = shared.pinned_bundle(sandbox, bundles) + if bundle.name != sandbox.harness_ref["name"] or bundle.agent != sandbox.agent: + raise shared.InstallerError("harness identity or agent mismatch") + return {"name": bundle.name, "digest": bundle.digest} + + +def configure(command, driver, bundle, workspace, sandbox, catalog): + """Add only the approved RO tree and retain the existing signed tool gate.""" + info = shared.describe_harness_tree(bundle_tree(bundle), inline=True) + attached = {p.type for p in workspace.providers if p.enabled and p.name in sandbox.providers} + shared.check_harness_governance(workspace, sandbox, info, catalog, attached) + config = shared.openclaw_harness_config(info) + ids = shared.harness_plugin_ids(info) + if any(not re.fullmatch(r"[a-zA-Z0-9_-]+", name) or name == "saw-tool-gate" for name in ids): + raise shared.InstallerError("invalid or reserved harness plugin ID") + policy = yaml.safe_load((Path(shared.__file__).parent / "tool-actions.yaml").read_text()) + tools = {r["tool"] for r in policy["rules"] if r["action"] in ("allow", "approval")} + discovery = {"tool_search", "tool_call"} + mcp_tools = sorted(t for t in tools if t in discovery or t.partition("__")[0] in info["mcpServers"]) + spec = {"trustEnv": shared.MCP_TRUST_ENV, "tools": mcp_tools, "digest": bundle.digest, "paths": config.get("plugins.load.paths", []), + "skills": config.get("skills.load.extraDirs", []), "plugins": ids, + "servers": shared.harness_mcp_servers(info)} + mounts = driver["containers"]["agent"]["volume_mounts"] + mounts.append({"name": "saw-gate", "mount_path": shared.HARNESS_MOUNT, "read_only": True, + "sub_path": "harness-" + bundle.digest.removeprefix("sha256:")}) + # The bundle's skill and writer agree on /sandbox/briefing. Retain that subtree with its workspace. + mounts.append({"name": "saw-workspace", "mount_path": "/sandbox/briefing", + "read_only": False, "sub_path": "briefing"}) + start = Path(__file__).with_name("harness_preflight.js").read_text() + start = start.replace("HARNESS_SPEC", json.dumps(spec, ensure_ascii=True)) + anchor = "const target = process.env.OPENCLAW_CONFIG_PATH;" + command[2] = command[2].replace(anchor, start + "\n" + anchor) + enable = "\n".join(shared.enable_harness_plugin(name) for name in ids) + command[2] = command[2].replace("exec openclaw gateway run", enable + "\nexec openclaw gateway run") diff --git a/kubernetes-runtime/saw_kubernetes/harness_preflight.js b/kubernetes-runtime/saw_kubernetes/harness_preflight.js new file mode 100644 index 000000000..b5b131fac --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/harness_preflight.js @@ -0,0 +1,21 @@ +// Uses treeFiles and mountOptions from the tool-gate preflight in the same startup command. +const harness = HARNESS_SPEC; +const harnessRoot = '/sandbox/harness'; +if (!mountOptions(harnessRoot).includes('ro')) throw Error('harness must have a read-only mount'); +const harnessHash = crypto.createHash('sha256'); +for (const path of treeFiles(harnessRoot).sort()) { + const hash = crypto.createHash('sha256').update(fs.readFileSync(path)).digest('hex'); + harnessHash.update(path.slice(harnessRoot.length + 1) + '\0' + hash + '\n'); +} +if ('sha256:' + harnessHash.digest('hex') !== harness.digest) throw Error('harness digest mismatch'); +config.plugins.load.paths.push(...harness.paths); +config.plugins.allow.push(...harness.plugins); +if (harness.skills.length) config.skills = {load: {extraDirs: harness.skills}}; +for (const entry of Object.values(harness.servers)) { + if (!entry.command) continue; + for (const name of harness.trustEnv) + if (process.env[name]) entry.env[name] = '${' + name + '}'; +} +config.tools.allow.push(...harness.tools); +config.channels = {...config.channels, slack: {enabled: false}}; +config.mcp = {servers: harness.servers}; diff --git a/kubernetes-runtime/saw_kubernetes/intent.py b/kubernetes-runtime/saw_kubernetes/intent.py new file mode 100644 index 000000000..93589de01 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/intent.py @@ -0,0 +1,195 @@ +"""Build a controller intent ConfigMap from local observed data; never apply it.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import sys +import uuid + +import yaml + +from .compiler import OWNER, canonical, compile_profile_files, digest +from .controller import KEY, MAX_BODY, MAX_DATA, NAME, valid_lifecycle_intent + + +class IntentError(ValueError): + """Messages are fixed classifications, never input snippets or credentials.""" + + +class _Parser(argparse.ArgumentParser): + def error(self, message): + raise IntentError("invalid command-line arguments") + + +class _UniqueSafeLoader(yaml.SafeLoader): + def construct_mapping(self, node, deep=False): + self.flatten_mapping(node) + result = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if key in result: + raise ValueError() + result[key] = self.construct_object(value_node, deep=deep) + return result + + +def _name(value): + return isinstance(value, str) and len(value) <= 253 and NAME.fullmatch(value) and ".." not in value + + +def _uuid(value): + try: + return isinstance(value, str) and str(uuid.UUID(value)) == value and uuid.UUID(value).int != 0 + except (ValueError, TypeError, AttributeError): + return False + + +def _object(value): + if not isinstance(value, dict) or value.get("apiVersion") != "v1" or value.get("kind") != "ConfigMap": + raise IntentError("input must be a v1 ConfigMap object") + meta = value.get("metadata", {}) + if not isinstance(meta, dict) or not _name(meta.get("name")) or not _name(meta.get("namespace")): + raise IntentError("ConfigMap name and namespace are required") + if meta.get("deletionTimestamp"): + raise IntentError("deleting ConfigMaps cannot supply an intent") + return meta + + +def _previous(value, namespace, name, owner): + meta = _object(value) + if (meta["namespace"] != namespace or meta["name"] != name + or meta.get("labels", {}).get(OWNER) != owner): + raise IntentError("previous intent belongs to another controller scope") + data = value.get("data", {}) + if value.get("binaryData") or not isinstance(data, dict) or set(data) != {"intent.json"}: + raise IntentError("previous intent has an invalid data envelope") + try: + intent = _json(data["intent.json"]) + except (ValueError, TypeError): + raise IntentError("previous intent contains invalid JSON") from None + if (not valid_lifecycle_intent(intent) + or not isinstance(intent["bindings"], dict) + or intent["bindings"].get("ownerID") != owner + or intent["bindings"].get("namespace") != namespace): + raise IntentError("previous intent has an invalid schema") + reference = intent["profileRef"] + if (not isinstance(reference, dict) or set(reference) not in ({"name", "uid", "sha256"}, {"name", "uid", "sha256", "planSHA256"}) + or ("planSHA256" in reference and not re.fullmatch(r"[0-9a-f]{64}", str(reference["planSHA256"]))) + or not _name(reference["name"]) or not _uuid(reference["uid"]) + or not isinstance(reference["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", reference["sha256"])): + raise IntentError("previous profile reference is invalid") + return intent + + +def build_intent(profile, bindings, *, name, generation, desired_state="Running", previous=None, pin_plan=False, restart_generation=None): + """Validate locally and return the controller's exact ConfigMap envelope. + + Observed metadata is required, but a local file is not authenticated cluster + evidence. The controller rechecks UID/hash/ownership against the live API. + `previous` optionally detects generation mistakes before publication. + """ + meta = _object(profile) + if not _uuid(meta.get("uid")) or not isinstance(meta.get("resourceVersion"), str) or not meta["resourceVersion"]: + raise IntentError("profile must include observed UID and resourceVersion") + if not _name(name) or name == meta["name"]: + raise IntentError("intent name must be valid and distinct from the profile") + if type(generation) is not int or generation < 1 or desired_state not in {"Running", "Stopped"}: + raise IntentError("a positive generation and Running or Stopped state are required") + if (not isinstance(bindings, dict) or bindings.get("namespace") != meta["namespace"] + or not _uuid(bindings.get("ownerID"))): + raise IntentError("administrator namespace and canonical owner must match the profile scope") + data = profile.get("data", {}) + if (profile.get("binaryData") or not isinstance(data, dict) or not data + or any(not isinstance(k, str) or not KEY.fullmatch(k) or k in {".", ".."} + or not isinstance(v, str) for k, v in data.items())): + raise IntentError("profile must contain string flat-file data only") + if len(canonical(data).encode()) > MAX_DATA: + raise IntentError("profile data exceeds the controller size bound") + try: + plan = compile_profile_files(data, bindings) + except Exception: + raise IntentError("profile or bindings are unsupported or invalid for this runtime") from None + intent = {"schemaVersion": 1, "generation": generation, "desiredState": desired_state, + "profileRef": {"name": meta["name"], "uid": meta["uid"], "sha256": digest(data)}, + "bindings": bindings} + if pin_plan: + intent["profileRef"]["planSHA256"] = plan["revision"] + old = _previous(previous, meta["namespace"], name, bindings["ownerID"]) if previous is not None else None + if restart_generation is None and old is not None: + restart_generation = old.get("restartGeneration") + if restart_generation is not None: + intent["restartGeneration"] = restart_generation + if not valid_lifecycle_intent(intent): + raise IntentError("invalid restart generation") + if old is not None: + if intent.get("restartGeneration", 0) < old.get("restartGeneration", 0): + raise IntentError("restart generation rolled back") + if old["profileRef"]["name"] != meta["name"] or old["profileRef"]["uid"] != meta["uid"]: + raise IntentError("profile identity changed; explicit controller recovery is required") + if old["generation"] > generation or old["generation"] == generation and old != intent: + raise IntentError("generation rolled back or was reused for different content") + encoded = canonical(intent) + if len(canonical({"intent.json": encoded}).encode()) > MAX_DATA: + raise IntentError("intent data exceeds the controller size bound") + return {"apiVersion": "v1", "kind": "ConfigMap", "metadata": { + "name": name, "namespace": meta["namespace"], "labels": {OWNER: bindings["ownerID"]}}, + "data": {"intent.json": encoded}} + + +def _json(raw): + def unique(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError() + value[key] = item + return value + return json.loads(raw, object_pairs_hook=unique) + + +def _read(path, *, json_only=False): + try: + with Path(path).open("rb") as stream: + raw = stream.read(MAX_BODY + 1) + if len(raw) > MAX_BODY: + raise IntentError("local input exceeds the size bound") + text = raw.decode("utf-8") + return _json(text) if json_only else yaml.load(text, Loader=_UniqueSafeLoader) + except IntentError: + raise + except (OSError, ValueError, yaml.YAMLError, RecursionError): + raise IntentError("cannot parse the local input file") from None + + +def main(argv=None): + parser = _Parser(description=__doc__) + parser.add_argument("--profile-configmap", required=True, help="Observed ConfigMap JSON, including metadata UID/resourceVersion") + parser.add_argument("--bindings", required=True, help="Administrator bindings YAML or JSON; no credential files are read") + parser.add_argument("--name", required=True, help="Allowlisted controller intent ConfigMap name") + parser.add_argument("--generation", required=True, type=int) + parser.add_argument("--desired-state", choices=["Running", "Stopped"], default="Running") + parser.add_argument("--restart-generation", type=int, help="Lifecycle restart generation; defaults to the previous value when updating") + parser.add_argument("--pin-plan", action="store_true", help="Require the deployed compiler to produce exactly this plan before applying") + parser.add_argument("--previous-intent", help="Optional existing intent ConfigMap JSON for update/generation checks") + try: + args = parser.parse_args(argv) + result = build_intent(_read(args.profile_configmap, json_only=True), _read(args.bindings), + name=args.name, generation=args.generation, desired_state=args.desired_state, pin_plan=args.pin_plan, restart_generation=args.restart_generation, + previous=_read(args.previous_intent, json_only=True) if args.previous_intent else None) + output = json.dumps(result, indent=2, sort_keys=True) + except IntentError as exc: + print(str(exc), file=sys.stderr) + return 1 + except Exception: + print("invalid controller intent inputs", file=sys.stderr) + return 1 + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kubernetes-runtime/saw_kubernetes/openclaw.sh b/kubernetes-runtime/saw_kubernetes/openclaw.sh new file mode 100644 index 000000000..cc69ea903 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/openclaw.sh @@ -0,0 +1,41 @@ +set -eu +umask 077 +export HOME=/sandbox/workspace OPENCLAW_HOME=/sandbox/workspace +export OPENCLAW_CONFIG_PATH=/sandbox/workspace/.openclaw/openclaw.json +export TMPDIR=/sandbox/workspace/.openclaw/state SQLITE_TMPDIR=/sandbox/workspace/.openclaw/state +mkdir -p "$TMPDIR" +# Only the stock gateway's injected placeholder is visible here. Keep its +# environment reference in the config so the supervisor can rotate it. +node - "$1" <<'JS' +const fs = require('node:fs'); +const crypto = require('node:crypto'); +const model = process.argv[2]; +const config = { + gateway: {mode: 'local', bind: 'loopback', port: 18789, + auth: {mode: 'token', token: crypto.randomBytes(32).toString('hex')}}, + models: {providers: {@SAW_PROVIDER_KEY@: {baseUrl: @SAW_BASE_URL@, + @SAW_TIMEOUT@api: '@SAW_API@', apiKey: '${@SAW_KEY_ENV@}', + models: [{id: model, name: model}]}}}, + agents: {defaults: {model: {primary: '@SAW_PROVIDER_ID@/' + model}, + workspace: '/sandbox/workspace/project'}}, + tools: {deny: ['*']} +}; +@SAW_BROWSER_CONFIG@ +const target = process.env.OPENCLAW_CONFIG_PATH; +// Remove previously managed harness/gate capabilities when absent from this +// approved startup. Do this after the preflights, never before their checks. +for (const [path, empty] of Object.entries({ + plugins:{allow:[], load:{paths:[]}, entries:{}}, skills:{load:{extraDirs:[]}}, + channels:{}, mcp:{servers:{}} +})) config[path] ??= empty; +// Publish complete managed subtrees with the stock writer. Replacing the config +// directly loses its metadata and triggers restoration of the previous backup. +const batch = target + '.saw-managed'; +const tmp = batch + '.' + crypto.randomUUID(); +fs.writeFileSync(tmp, JSON.stringify(Object.entries(config).map(([path, value]) => ({path, value}))), + {flag: 'wx', mode: 0o600}); +fs.renameSync(tmp, batch); +JS +openclaw config set --batch-file "$OPENCLAW_CONFIG_PATH.saw-managed" +rm -f "$OPENCLAW_CONFIG_PATH.saw-managed" +exec openclaw gateway run --bind loopback --port 18789 diff --git a/kubernetes-runtime/saw_kubernetes/portal_contract.py b/kubernetes-runtime/saw_kubernetes/portal_contract.py new file mode 100644 index 000000000..76dbf3f8e --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/portal_contract.py @@ -0,0 +1,80 @@ +"""Reusable owner-specific approval of a previously reviewed OpenShift boundary. + +Grant no permissions here. Enrollment runs with named, read-only API access. +The normal runtime verifier still pins object UIDs and the full, unmodified +resource hashes after each registration; this comparison is the prerequisite. +""" +import copy +import hashlib +import ipaddress +import json +import re +import ssl +from .boundary import BoundaryError, fingerprint, objects + + +def contract(config, get): + ns, trusted, owner = config["namespace"], config["trustedNamespace"], config["ownerID"] + if not re.fullmatch(r"saw-ctl-[0-9a-f]{16}", trusted): + raise BoundaryError("InvalidControlNamespace") + output = {} + for key,(path,field) in objects(config).items(): + obj=get(path) + value=copy.deepcopy(obj[field]) + if key.startswith("governance"): + # Cluster installation is shared across registrations. Keep both UID and hash. + output[key]={"uid":obj["metadata"]["uid"],"sha256":fingerprint(value)} + continue + if key=="publicCA": + source=get(f"/api/v1/namespaces/{trusted}/configmaps/{config['gateway']}-proxy-ca") + if obj.get("metadata",{}).get("labels",{}).get("saw.redhat.com/runtime-owner")!=owner or set(value)!={"ca.crt"} or value!=source.get("data") or source["metadata"].get("deletionTimestamp"): + raise BoundaryError("UntrustedProxyCA") + cert=value["ca.crt"] + if not re.fullmatch(r"-----BEGIN CERTIFICATE-----\s+[A-Za-z0-9+/=\s]+-----END CERTIFICATE-----\s*",cert): + raise BoundaryError("InvalidProxyCA") + try: + ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=cert) + except (ssl.SSLError,ValueError) as exc: + raise BoundaryError("InvalidProxyCA") from exc + output[key]="equals independent trusted proxy public CA" + continue + if key=="gatewayService": + ips=value.get("clusterIPs") + if (value.get("type")!="ClusterIP" or not isinstance(ips,list) or not ips or value.get("clusterIP")!=ips[0] + or [("IPv4" if ipaddress.ip_address(ip).version==4 else "IPv6") for ip in ips]!=value.get("ipFamilies")): + raise BoundaryError("InvalidServiceAllocation") + # Only API-assigned addresses vary. Exact ports, selectors, type, + # ipFamilies, traffic policies and all remaining fields are pinned. + del value["clusterIP"],value["clusterIPs"] + if key=="gateway": + annotations=value["template"]["metadata"].get("annotations",{}) + if not re.fullmatch("[0-9a-f]{64}",annotations.get("checksum/config","")): + raise BoundaryError("MissingConfigRolloutMarker") + # The config itself is approved under gatewayConfig. + del annotations["checksum/config"] + text=json.dumps(value,sort_keys=True,separators=(",",":")) + if "@SAW_" in text: + raise BoundaryError("ReservedApprovalMarker") + # Only substitutions produced by this owner's template are accepted. + # The runtime namespace, owner login and OIDC issuer remain literal. + text=text.replace(trusted,"@SAW_CONTROL_NS@").replace(owner,"@SAW_INSTALLATION@") + if key == "validation": + slots = config.get("slotKeys", []) + registration = trusted.removeprefix("saw-ctl-") + for slot in slots: + relay = hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] if len(slots) > 1 else registration + # Substitute only registered name forms. Keep slot, CA and spec + # checks literal, including the association between Job and PVC. + if len(slots) > 1: + text = text.replace(config["gateway"] + "-" + relay[:8], "@SAW_SLOT_" + slot + "@") + text = text.replace("gate-" + relay + "-", "gate-@SAW_RELAY_" + slot + "@-") + text = text.replace("ui-" + relay + "-browser-proof", "ui-@SAW_RELAY_" + slot + "@-browser-proof") + output[key]=fingerprint(json.loads(text)) + return output + + +def verify_contract(config, get, expected): + observed=contract(config,get) + for key in set(observed)|set(expected): + if observed.get(key)!=expected.get(key): + raise BoundaryError("UnapprovedRegistrationResource:"+key) diff --git a/kubernetes-runtime/saw_kubernetes/prepare_audit.py b/kubernetes-runtime/saw_kubernetes/prepare_audit.py new file mode 100644 index 000000000..c48737cec --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/prepare_audit.py @@ -0,0 +1,46 @@ +"""Seal the workspace's dedicated, pre-mounted audit file on a Linux filesystem.""" +import array +import fcntl +import os +import stat +import sys + +# linux/fs.h defines the argument type as long, including on 64-bit Linux. +LONG_SIZE = array.array("l").itemsize +FS_IOC_GETFLAGS = 0x80006601 | (LONG_SIZE << 16) +FS_IOC_SETFLAGS = 0x40006602 | (LONG_SIZE << 16) +FS_APPEND_FL = 0x20 + + +def prepare(root): + directory = os.open(root, os.O_DIRECTORY | os.O_NOFOLLOW) + try: + fd = os.open("tool-actions.jsonl", os.O_APPEND | os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW + | os.O_NONBLOCK, 0o600, dir_fd=directory) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.getuid(): + raise RuntimeError("audit target requires a single regular file owned by the workspace UID") + flags = array.array("l", [0]) + fcntl.ioctl(fd, FS_IOC_GETFLAGS, flags, True) + if not flags[0] & FS_APPEND_FL: + if info.st_size: + raise RuntimeError("refusing to adopt a nonempty unsealed log") + flags[0] |= FS_APPEND_FL + fcntl.ioctl(fd, FS_IOC_SETFLAGS, flags) + fcntl.ioctl(fd, FS_IOC_GETFLAGS, flags, True) + if not flags[0] & FS_APPEND_FL: + raise RuntimeError("filesystem failed to seal the log") + os.fsync(fd) + finally: + os.close(fd) + os.fsync(directory) + finally: + os.close(directory) + + +if __name__ == "__main__": + root = sys.argv[1] + os.makedirs(root, mode=0o750, exist_ok=True) + prepare(root) + print("audit inode is append-only") diff --git a/kubernetes-runtime/saw_kubernetes/prepare_gate.py b/kubernetes-runtime/saw_kubernetes/prepare_gate.py new file mode 100644 index 000000000..5d382c89c --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/prepare_gate.py @@ -0,0 +1,140 @@ +"""Publish the shared tool-gate tree on its dedicated PVC before workspace startup.""" +import argparse +import fcntl +import json +import re +import os +from pathlib import Path +import tempfile + +from .core import shared + + +def verify(directory, files, model_selector=False, browser=False): + if directory.is_symlink() or not directory.is_dir(): + raise ValueError("invalid gate directory") + dirs = {p for item in files for p in Path(item).parents} - {Path(".")} + for path in directory.rglob("*"): + rel = path.relative_to(directory) + if browser and rel.as_posix() == "relay-proof" and not path.is_symlink() and path.is_file(): + continue + if model_selector and rel.as_posix()=="model.json" and path.is_file() and not path.is_symlink(): + continue + if path.is_symlink() or (rel not in dirs if path.is_dir() else rel.as_posix() not in files): + raise ValueError("unexpected entry in the trusted gate") + for rel, (data, _) in files.items(): + path = directory / rel + if not path.is_file() or path.read_bytes() != data: + raise ValueError("existing gate differs from the requested policy or owner") + + +def publish(root, identity, receipt_key=None, *, model=None, generation=None, browser=False): + assets = Path(shared.__file__).parent + files, _, digest, _ = shared.tool_gate_bundle(assets, identity, receipt_key) + target = root / "bundle" + publish_tree(root, target, files, model_selector=model is not None, browser=browser) + if model is not None: + publish_model(root, target, model, generation) + return digest + + +def publish_tree(root, target, files, *, model_selector=False, browser=False): + if not target.exists(): + with tempfile.TemporaryDirectory(dir=root) as scratch: + tmp = Path(scratch) / "bundle" + tmp.mkdir(mode=0o755) + for rel, (data, _) in files.items(): + path = tmp / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(0o444) + # The final nonempty tree is never overwritten, including concurrent publishers. + try: + tmp.rename(target) + except FileExistsError: + pass + except OSError: + if not target.is_dir(): + raise + verify(target, files, model_selector=model_selector, browser=browser) + + +def publish_harness(root, bom, name, expected_digest): + from .harness import bundle_tree, read_bundles + _, raw, index = shared.read_profile_files(bom) + bundle = read_bundles(raw, index).get(name) + if not bundle or bundle.name != name or bundle.digest != expected_digest: + raise ValueError("mounted BOM does not match enrolled harness") + publish_tree(root, root / ("harness-" + bundle.digest.removeprefix("sha256:")), bundle_tree(bundle)) + + +def publish_model(root, target, model, generation): + """Fence concurrent or replayed GitOps Jobs on the per-sandbox gate volume.""" + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,99}", model) or type(generation) is not int or generation < 1: + raise ValueError("invalid model selection") + selected = {"generation":generation, "model":model} + path = target / "model.json" + with (root / ".model-lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if path.exists(): + previous = json.loads(path.read_text()) + if previous["generation"] > generation: + return # A newer publication already won. + if previous["generation"] == generation: + if previous != selected: + raise ValueError("conflicting model at the same generation") + return + with tempfile.NamedTemporaryFile(dir=root, mode="w", delete=False) as tmp: + try: + json.dump(selected, tmp) + tmp.flush() + os.fchmod(tmp.fileno(), 0o444) + os.fsync(tmp.fileno()) + os.replace(tmp.name, path) + finally: + Path(tmp.name).unlink(missing_ok=True) + + +def publish_browser_proof(directory, proof): + if not re.fullmatch(rb"[a-f0-9]{64}", proof): + raise ValueError("invalid browser proof") + path = directory / "relay-proof" + with tempfile.NamedTemporaryFile(dir=directory, delete=False) as tmp: + try: + tmp.write(proof) + tmp.flush() + os.fchmod(tmp.fileno(), 0o444) + os.fsync(tmp.fileno()) + try: + os.link(tmp.name, path) + except FileExistsError: + if path.is_symlink() or path.read_bytes() != proof: + raise ValueError("existing browser proof differs") + finally: + Path(tmp.name).unlink(missing_ok=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + for key in ("owner", "subject", "workspace", "sandbox"): + parser.add_argument("--" + key, required=True) + parser.add_argument("--harness", nargs=2, metavar=("NAME", "DIGEST")) + parser.add_argument("--browser-proof") + parser.add_argument("--model") + parser.add_argument("--model-generation",type=int) + receipt = parser.add_mutually_exclusive_group() + receipt.add_argument("--receipt-key") + receipt.add_argument("--writer-service") + parser.add_argument("--writer-namespace") + parser.add_argument("--writer-ca") + args = parser.parse_args() + if args.writer_service: + from .receipt import receipt_key + args.receipt_key = receipt_key(args.writer_service, args.writer_namespace, args.writer_ca) + os.umask(0o022) + if args.harness: + publish_harness(Path("/volume"), Path("/bom"), *args.harness) + result = publish(Path("/volume"), shared.tool_gate_identity(args.owner, args.subject, args.workspace, args.sandbox), args.receipt_key, model=args.model, generation=args.model_generation, browser=bool(args.browser_proof)) + if args.browser_proof: + publish_browser_proof(Path("/volume/bundle"), Path(args.browser_proof).read_bytes()) + print(result) diff --git a/kubernetes-runtime/saw_kubernetes/provider.py b/kubernetes-runtime/saw_kubernetes/provider.py new file mode 100644 index 000000000..c917691b6 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/provider.py @@ -0,0 +1,101 @@ +"""Verify Kubernetes proxy bindings against the same shipped governance profiles as the VM.""" +import re +import ipaddress +from functools import lru_cache +from pathlib import Path +from urllib.parse import urlsplit + +import yaml + +from .core import shared + +PROFILES = Path(shared.__file__).resolve().parents[1] / "provider-profiles" +SUPPORTED = {"openai", "anthropic", "nvidia", "slack", "gmail", "brave"} + + +@lru_cache(maxsize=5) +def document(kind): + if kind not in SUPPORTED: + raise ValueError("unsupported governed provider") + return yaml.safe_load((PROFILES / (kind + ".yaml")).read_text()) + + +def custom_host(url): + """The namespace credential broker intercepts only HTTPS on port 443.""" + if not isinstance(url, str) or shared.check_base_url(url) != url: + raise ValueError("invalid custom inference base URL") + parsed = urlsplit(url) + hostname = parsed.hostname or "" + try: + ipaddress.ip_address(hostname) + except ValueError: + pass + else: + raise ValueError("credential broker requires a DNS hostname") + label = r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?" + if (parsed.scheme != "https" or parsed.port not in (None, 443) + or len(hostname) > 253 or not re.fullmatch(rf"{label}(?:\.{label})+", hostname) + or not re.fullmatch(r"(?:/[A-Za-z0-9_-]+)*/v1", parsed.path)): + raise ValueError("custom inference requires an approved HTTPS v1 endpoint on port 443") + return parsed.hostname + + +def requirements(kind, custom_url=None): + if kind == "custom-inference" and custom_url: + return {"name":"api_key", "env_vars":["CUSTOM_INFERENCE_API_KEY"]}, [custom_host(custom_url)] + doc = document(kind) + slot, = doc["credentials"] + hosts = sorted({e["host"] for e in doc["endpoints"]}) + return slot, hosts + + +def checked_marker(provider, bindings, custom_url=None): + """The plan carries only the public routing marker; keys remain in the proxy.""" + slot, _ = requirements(provider.type, custom_url) + expected = {"secret": provider.credential_secret, "key": provider.credential_secret_key, + "type": provider.type, "refresh": bool(provider.refresh_strategy)} + if (expected not in bindings or provider.credentials_spec or provider.base_url or provider.base_url_secret_key + or provider.refresh_strategy not in ("", "oauth2_refresh_token") or provider.inference_timeout + or (provider.type != "nvidia" and (provider.model or provider.nemoclaw_provider)) + or provider.model_secret_key): + raise ValueError("provider configuration differs from enrolled proxy binding") + if expected["refresh"] and (slot.get("refresh") or {}).get("strategy") != provider.refresh_strategy: + raise ValueError("provider refresh is not supported by its governance profile") + if not re.fullmatch(r"[a-z0-9-]+", expected["secret"]) or not re.fullmatch(r"[a-zA-Z0-9_-]+", expected["key"]): + raise ValueError("invalid proxy marker reference") + return {slot["env_vars"][0]: shared.credential_marker("saw-cred", expected["secret"], expected["key"])} + + +def check_policy(kind, credentials, policy, custom_url=None): + slot, hosts = requirements(kind, custom_url) + if kind == "custom-inference": + # Same contract as the VM: the trusted Secret's url supplies the marker + # host. No static profile may add hosts, even under another binding. + return (policy.get("markerPrefix") == "saw-cred" + and not policy["profiles"].get(kind) + and policy["allowHosts"] == hosts + and len(policy["bindings"]) == 1 + and any(b.get("providers") == [kind] and b.get("key") == "api_key" + and not b.get("refresh") and credentials == {slot["env_vars"][0]: + shared.credential_marker("saw-cred", b["secret"], b["key"])} for b in policy["bindings"])) + + if (policy.get("markerPrefix") != "saw-cred" or sorted(policy["profiles"].get(kind, [])) != hosts or + not set(hosts) <= set(policy["allowHosts"])): + return False + for binding in policy["bindings"]: + if kind not in binding["providers"]: + continue + if set(binding["providers"]) - ({kind, "build"} if kind=="nvidia" else {kind}): + continue + marker = shared.credential_marker("saw-cred", binding["secret"], binding["key"]) + if credentials != {slot["env_vars"][0]: marker}: + continue + refresh = binding.get("refresh") + if refresh: + contract = slot.get("refresh", {}) + if (refresh.get("strategy") != contract.get("strategy") or + refresh.get("tokenURL") != contract.get("token_url") or + refresh.get("material") != {"clientID":"client_id", "clientSecret":"client_secret", "refreshToken":"refresh_token"}): + return False + return True + return False diff --git a/kubernetes-runtime/saw_kubernetes/receipt.py b/kubernetes-runtime/saw_kubernetes/receipt.py new file mode 100644 index 000000000..2505f5be9 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/receipt.py @@ -0,0 +1,22 @@ +"""Fetch only the writer's Ed25519 public receipt key using its service identity.""" +import base64 +import re +import ssl +import urllib.request + + +def receipt_key(service, namespace, ca_file, *, opener=None): + label = r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?" + if not all(re.fullmatch(label, s) for s in (service, namespace)): + raise ValueError("invalid writer service") + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args): + raise ValueError("writer redirect refused") + opener = opener or urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect(), + urllib.request.HTTPSHandler(context=ssl.create_default_context(cafile=ca_file))) + with opener.open(f"https://{service}.{namespace}.svc:8443/v1/receipt-key", timeout=8) as response: + raw = response.read(65) + der = base64.b64decode(raw, validate=True) + if len(der) != 44 or der[:12] != bytes.fromhex("302a300506032b6570032100"): + raise ValueError("expected Ed25519 SubjectPublicKeyInfo") + return raw.decode("ascii") diff --git a/kubernetes-runtime/saw_kubernetes/reconcile.py b/kubernetes-runtime/saw_kubernetes/reconcile.py new file mode 100644 index 000000000..04bcbd073 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/reconcile.py @@ -0,0 +1,469 @@ +"""Serialized, restart-safe reconciliation over the stock OpenShell RPC surface. + +The adapter implements ``call(method, request) -> response`` using protobuf-field +names. It must not retry unsafe writes itself or log request/response bodies. +""" + +from __future__ import annotations + +import contextlib +import copy +import fcntl +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace + +from .core import shared +from . import provider as governed + +from .compiler import ContractError, MANAGED, OWNER, REVISION, SOURCE_REVISION, digest, request_id + + +class RpcError(Exception): + def __init__(self, code): + # Do not persist arbitrary gateway errors that could contain credentials. + self.code = code if code in { + "NOT_FOUND", "ALREADY_EXISTS", "UNAVAILABLE", "DEADLINE_EXCEEDED", + "ABORTED", "RESOURCE_EXHAUSTED", "PERMISSION_DENIED", "UNAUTHENTICATED", + "INVALID_ARGUMENT", "FAILED_PRECONDITION", "UNKNOWN"} else "UNKNOWN" + super().__init__(self.code) + + +class MissingSandboxReceipt(ContractError): + """Stopping an ambiguous create requires a verified receipt from a Running pass.""" + + +class Busy(Exception): + pass + + +class StateStore: + """Single local/shared-filesystem writer; not a distributed Kubernetes lease.""" + + def __init__(self, directory): + self.directory = Path(directory) + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + if self.directory.is_symlink(): + raise ContractError("state directory must not be a symlink", safe_reason="ReconcilerStateRejected") + self.path = self.directory / "state.json" + + @contextlib.contextmanager + def locked(self): + fd = os.open(self.directory / "reconcile.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Busy() from None + yield + finally: + os.close(fd) + + def load(self): + if not self.path.exists(): + return {} + if self.path.is_symlink(): + raise ContractError("state must not be a symlink", safe_reason="ReconcilerStateRejected") + try: + value = json.loads(self.path.read_text()) + except (ValueError, OSError): + raise ContractError("cannot read retained reconciler state", + safe_reason="ReconcilerStateRejected") from None + if not isinstance(value, dict): + raise ContractError("retained reconciler state must be an object", + safe_reason="ReconcilerStateRejected") + return value + + def save(self, value): + fd, tmp = tempfile.mkstemp(prefix="state-", dir=self.directory) + try: + with os.fdopen(fd, "w") as stream: + json.dump(value, stream, sort_keys=True) + stream.flush(); os.fsync(stream.fileno()) + os.replace(tmp, self.path) + dirfd = os.open(self.directory, os.O_RDONLY) + try: + os.fsync(dirfd) + finally: + os.close(dirfd) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +class Reconciler: + """One bounded observation/apply pass; callers schedule the next pass. + + `verify_resources(plan)` must independently read/validate current cluster + resource identity and integration security prerequisites. It is mandatory, + called before every pass, and must fail closed. Tests use fake resources; + providing a no-op in a deployment is not qualification. + """ + + def __init__(self, rpc, state_directory, verify_resources, *, verify_shutdown=None): + self.rpc = rpc + self.store = StateStore(state_directory) + if not callable(verify_resources): + raise ContractError("resource verifier is required", safe_reason="InvalidPlan") + self.verify_resources = verify_resources + self.verify_shutdown = (verify_shutdown if verify_shutdown is not None else + getattr(verify_resources, "verify_shutdown", verify_resources)) + if not callable(self.verify_shutdown): + raise ContractError("shutdown identity verifier must be callable", safe_reason="InvalidPlan") + + def _get(self, method, request, field): + try: + response = self.rpc.call(method, request) + except RpcError as exc: + if exc.code == "NOT_FOUND": + return None + raise + if not isinstance(response.get(field), dict): + raise ContractError("gateway returned an invalid object", safe_reason="GatewayMismatch") + return response[field] + + def _mutate(self, method, request, plan, generation): + body = copy.deepcopy(request) + body["request_id"] = request_id(plan["ownerID"], method, + {"request": request, "generation": generation}) + try: + return self.rpc.call(method, body) + except RpcError as exc: + if exc.code != "ALREADY_EXISTS": + raise + # Observe on the next pass, including ALREADY_EXISTS and ambiguous + # timeouts. Never assume the mutation response proves convergence. + + def _identity(self, kind, workspace, name, obj, plan, state, default=False): + meta = obj.get("metadata", {}) + labels = meta.get("labels", {}) + if meta.get("deletion_time"): + raise ContractError("managed object is being deleted", safe_reason="IdentityConflict") + if meta.get("name") != name or not meta.get("id"): + raise ContractError("gateway object identity is missing or changed", safe_reason="IdentityConflict") + if kind != "workspace" and meta.get("workspace", "default") != workspace: + raise ContractError("gateway object is in the wrong workspace", safe_reason="IdentityConflict") + if not default and (labels.get(OWNER) != plan["ownerID"] or labels.get(MANAGED) != "kubernetes-runtime"): + raise ContractError("refusing to adopt an object owned by another installation", + safe_reason="IdentityConflict") + key = f"{kind}/{workspace}/{name}" + previous = state.setdefault("objectIDs", {}).get(key) + if previous is not None and previous != meta["id"]: + raise ContractError("same-name replacement detected; explicit recovery is required", + safe_reason="IdentityConflict") + state["objectIDs"][key] = meta["id"] + self.store.save(state) + + @staticmethod + def _sandbox_spec_matches(actual, desired): + # The pinned gateway adds provider_attachment_epoch and normalizes an + # unspecified restart policy to Never during create (grpc/sandbox.rs). + # Policy is governed by the platform, not compiled by this adapter; + # do not confuse that policy with permission to add other spec fields. + allowed = {"template", "providers", "command", "tty", "policy", + "provider_attachment_epoch", "restart_policy", "environment", + "resource_requirements", "log_level"} + if not isinstance(actual, dict) or set(actual) - allowed: + return False + if any(actual.get(field) for field in ["environment", "resource_requirements", "log_level"]): + return False + if actual.get("restart_policy", "SANDBOX_RESTART_POLICY_NEVER") != "SANDBOX_RESTART_POLICY_NEVER": + return False + for field in ["template", "providers", "command", "tty"]: + default = False if field == "tty" else ([] if field == "providers" else None) + if actual.get(field, default) != desired[field]: + return False + return True + + def reconcile(self, plan, *, generation, desired_state="Running"): + if type(generation) is not int or generation < 1 or desired_state not in {"Running", "Stopped"}: + raise ContractError("a positive intent generation and Running/Stopped state are required", + safe_reason="InvalidIntent") + body = {k: v for k, v in plan.items() if k != "revision"} + if plan.get("sourceRevision") != SOURCE_REVISION or plan.get("schemaVersion") != 1 or digest(body) != plan.get("revision"): + raise ContractError("invalid or modified compiled plan", safe_reason="InvalidPlan") + if plan.get("mode") not in ("container", "kata") or plan.get("requestedPlacement") != { + "supervisor": "container", "workload": plan["mode"]}: + raise ContractError("stock adapter requires an ordinary-container supervisor", + safe_reason="PlacementRejected") + with self.store.locked(): + state = self.store.load() + identity = {key: plan[key] for key in ["ownerID", "namespaceUID", "ownerSubject", "mode"]} + if state and state.get("identity") != identity: + raise ContractError("workspace identity, owner and deployment mode are immutable", + safe_reason="IdentityConflict") + previous = state.get("intent", {}) + intent = {"generation": generation, "revision": plan["revision"], "desiredState": desired_state} + if previous.get("generation", 0) > generation or (previous.get("generation") == generation and previous != intent): + raise ContractError("stale generation or conflicting intent at the same generation", + safe_reason="StaleIntent") + (self.verify_shutdown if desired_state == "Stopped" else self.verify_resources)(plan) + state.update(identity=identity, intent=intent) + self.store.save(state) + try: + return self._pass(plan, state, generation, desired_state) + except RpcError as exc: + attempts = min(state.get("transientAttempts", 0) + 1, 8) + state["transientAttempts"] = attempts + self.store.save(state) + if exc.code in {"UNAVAILABLE", "DEADLINE_EXCEEDED", "ABORTED", "RESOURCE_EXHAUSTED"}: + return self._status(plan, generation, "Retry", reason=exc.code, + retry_after_seconds=min(2 ** attempts, 60)) + raise + + @staticmethod + def _status(plan, generation, phase, **extra): + return {"observedGeneration": generation, "desiredRevision": plan["revision"], + "phase": phase, "requestedPlacement": plan["requestedPlacement"], **extra} + + def _pass(self, plan, state, generation, desired_state): + info = self.rpc.call("GetGatewayInfo", {}) + if info.get("gateway_version") != plan["gatewayVersion"] or [d.get("name") for d in info.get("compute_drivers", [])] != ["kubernetes"]: + raise ContractError("gateway version or active compute driver differs from approved inputs", + safe_reason="GatewayMismatch") + if desired_state == "Stopped": + return self._stop_pass(plan, state, generation) + retired = plan.get("retiredSandboxes", {}) + if retired: + refs = {"sandbox/" + name for name in retired} + if any(state.get("objectIDs", {}).get("sandbox/" + k) != v for k, v in retired.items()): + raise ContractError("refusing to stop a sandbox without its recorded identity", + safe_reason="SandboxStateMismatch") + pending = self._stop_references(refs, plan, state, generation) + if pending: + return self._status(plan, generation, "Pending", pending=pending, retry_after_seconds=5) + for provider_type in sorted({p["type"] for w in plan["workspaces"] for p in w["providers"]}): + # Resolve platform scope explicitly (workspace_scope omitted). A + # workspace administrator's same-name custom profile is not enough. + profile = self.rpc.call("GetProviderProfile", {"id": provider_type}).get("profile", {}) + marker_provider = any(p.get("credentials") for w in plan["workspaces"] for p in w["providers"] + if p["type"] == provider_type) + if marker_provider: + slots = profile.get("credentials", []) + endpoints = profile.get("endpoints", []) + slot, hosts = governed.requirements(provider_type, plan.get("customInferenceURL")) + if (profile.get("id") != provider_type + or profile.get("source") != "interceptor/governance" or len(slots) != 1 + or slots[0].get("env_vars") != slot["env_vars"] or slots[0].get("name") != slot["name"] + or sorted(e.get("host") for e in endpoints) != hosts + or any(e.get("port") != 443 or e.get("enforcement") != "NETWORK_ENFORCEMENT_MODE_ENFORCE" for e in endpoints)): + raise ContractError("provider profile differs from the approved inference boundary", + safe_reason="ProviderProfileMismatch") + continue + if (profile.get("id") != provider_type or profile.get("credentials") + or not profile.get("source", "").startswith("interceptor/")): + raise ContractError("provider profile is not a governed credential-free platform profile", + safe_reason="ProviderProfileMismatch") + pending = [] + adapter = SimpleNamespace( + prepare_workspace=lambda ws: self._prepare_workspace(ws, plan, state, generation, pending), + apply_provider=lambda ws, provider: self._apply_provider(ws, provider, plan, state, generation, pending), + apply_sandbox=lambda ws, sandbox: self._apply_sandbox( + ws, sandbox, plan, state, generation, desired_state, pending)) + for ws in plan["workspaces"]: + shared.apply_workspace_sequence(ws, ws["providers"], ws["sandboxes"], adapter) + state["transientAttempts"] = 0 + self.store.save(state) + return self._status(plan, generation, "Pending" if pending else + ("GenericReady" if desired_state == "Running" else "Stopped"), + pending=pending, retry_after_seconds=5 if pending else 0) + + def _stop_pass(self, plan, state, generation): + """Only stop previously observed IDs; never create or repair dependencies. + + Include retained sandboxes removed from current desired membership. + Revoked PVC approval or a failed policy prerequisite must not require + restarting a workload merely to regain the ability to stop it. + """ + ids = state.get("objectIDs", {}) + references = {f"sandbox/{ws['name']}/{sb['name']}" + for ws in plan["workspaces"] for sb in ws["sandboxes"]} + references.update(key for key in ids if key.startswith("sandbox/")) + pending = self._stop_references(references, plan, state, generation) + state["transientAttempts"] = 0 + self.store.save(state) + return self._status(plan, generation, "Pending" if pending else "Stopped", + pending=pending, retry_after_seconds=5 if pending else 0) + + def _stop_references(self, references, plan, state, generation): + ids = state.get("objectIDs", {}) + pending = [] + for ref in sorted(references): + _, workspace, name = ref.split("/") + scope = {"workspace_scope": {"workspace": workspace}} + sandbox = self._get("GetSandbox", {**scope, "name": name}, "sandbox") + if sandbox is None: + continue # No resource is created to satisfy Stopped intent. + workspace_obj = self._get("GetWorkspace", {"name": workspace}, "workspace") + workspace_ref = f"workspace//{workspace}" + if ref not in ids and ref in state.get("pendingSandboxCreates", {}): + raise MissingSandboxReceipt("start to recover the sandbox creation receipt before stopping", + safe_reason="SandboxStateMismatch") + if (ref not in ids or workspace_ref not in ids or workspace_obj is None + or sandbox.get("metadata", {}).get("id") != ids[ref] + or workspace_obj.get("metadata", {}).get("id") != ids[workspace_ref]): + raise ContractError("shutdown refuses unrecorded or replaced workspace/sandbox identity", + safe_reason="IdentityConflict") + self._identity("workspace", "", workspace, workspace_obj, plan, state, default=workspace == "default") + self._identity("sandbox", workspace, name, sandbox, plan, state) + phase = sandbox.get("status", {}).get("phase") + if phase == "SANDBOX_PHASE_STOPPED": + continue + if phase not in {"SANDBOX_PHASE_READY", "SANDBOX_PHASE_ERROR"}: + pending.append(ref) + continue + self._mutate("StopSandbox", {**scope, "name": name}, plan, generation) + pending.append(ref) + return pending + + def _prepare_workspace(self, ws, plan, state, generation, pending): + scope = {"workspace_scope": {"workspace": ws["name"]}} + obj = self._get("GetWorkspace", {"name": ws["name"]}, "workspace") + if obj is None: + if ws["name"] == "default": + raise ContractError("gateway default workspace is missing", safe_reason="WorkspaceRejected") + self._mutate("CreateWorkspace", {"name": ws["name"], "labels": { + MANAGED: "kubernetes-runtime", OWNER: plan["ownerID"]}}, plan, generation) + pending.append(f"workspace/{ws['name']}") + return False + self._identity("workspace", "", ws["name"], obj, plan, state, default=ws["name"] == "default") + if obj.get("status", {}).get("phase") != "WORKSPACE_PHASE_ACTIVE": + pending.append(f"workspace/{ws['name']}") + return False + members, token, tokens = [], "", set() + while True: + response = self.rpc.call("ListWorkspaceMembers", {**scope, "page_size": 100, "page_token": token}) + members.extend(response.get("members", [])) + token = response.get("next_page_token", "") + if not token: + break + if token in tokens or len(tokens) >= 100: + raise ContractError("invalid membership pagination", safe_reason="WorkspaceRejected") + tokens.add(token) + member = next((m for m in members if m.get("principal_subject") == plan["ownerSubject"]), None) + if member is None: + self._mutate("AddWorkspaceMember", {**scope, "principal_subject": plan["ownerSubject"], + "role": "WORKSPACE_ROLE_ADMIN"}, plan, generation) + pending.append(f"membership/{ws['name']}") + return False + if member.get("role") != "WORKSPACE_ROLE_ADMIN": + raise ContractError("existing owner membership has unexpected role", + safe_reason="WorkspaceRejected") + return True + + def _apply_provider(self, ws, provider, plan, state, generation, pending): + scope = {"workspace_scope": {"workspace": ws["name"]}} + pname = provider["metadata"]["name"] + reference = f"{ws['name']}/{pname}" + existing = self._get("GetProvider", {**scope, "name": pname}, "provider") + # Persist intent before the RPC: a committed create can outlive the + # connection (or this process). The pinned gateway durably replays the + # same caller/request ID/payload and checks the recorded object revision. + # A subsequent GetProvider, even with our labels, is not a create receipt. + creations = state.setdefault("pendingProxyProviders", {}) + intent = creations.get(reference) + if intent is not None and (not isinstance(intent, dict) + or intent.get("request") != digest(provider) + or type(intent.get("generation")) is not int or intent["generation"] < 1): + raise ContractError("pending provider creation changed; explicit recovery is required", + safe_reason="ProviderRejected") + if existing is None or intent is not None: + if provider.get("credentials") and intent is None: + intent = {"request": digest(provider), "generation": generation} + creations[reference] = intent + self.store.save(state) + reply = self._mutate("CreateProvider", {**scope, "provider": provider}, plan, + intent["generation"] if intent else generation) + if provider.get("credentials") and reply: + created = reply.get("provider", {}) + self._identity("provider", ws["name"], pname, created, plan, state) + meta = created["metadata"] + version = meta.get("resource_version") + if not version: + raise ContractError("missing provider revision", safe_reason="ProviderRejected") + state.setdefault("proxyProviderRevisions", {})[reference] = { + "version": version, "request": digest(provider)} + creations.pop(reference, None) + self.store.save(state) + elif existing is not None: + raise ContractError("provider creation receipt unavailable; explicit recovery is required", + safe_reason="ProviderRejected") + if existing is None: + pending.append(f"provider/{reference}") + return False + self._identity("provider", ws["name"], pname, existing, plan, state) + expected_credentials = {} + if provider.get("credentials"): + recorded = state.get("proxyProviderRevisions", {}).get(f"{ws['name']}/{pname}") + if recorded != {"version": existing["metadata"].get("resource_version"), "request": digest(provider)}: + raise ContractError("proxy provider ownership or credential revision requires explicit recovery", + safe_reason="ProviderRejected") + expected_credentials = dict.fromkeys(provider["credentials"], "REDACTED") + if (existing.get("type") != provider["type"] or existing.get("config") or existing.get("profile_workspace") + or existing.get("credentials", {}) != expected_credentials or existing.get("credential_handles")): + raise ContractError("provider drift or unexpected credential state requires explicit recovery", + safe_reason="ProviderRejected") + return True + + def _apply_sandbox(self, ws, sandbox, plan, state, generation, desired_state, pending): + scope = {"workspace_scope": {"workspace": ws["name"]}} + sname = sandbox["name"] + ref = f"sandbox/{ws['name']}/{sname}" + existing = self._get("GetSandbox", {**scope, "name": sname}, "sandbox") + creates = state.setdefault("pendingSandboxCreates", {}) + pending_create = creates.get(ref) + if existing is None or pending_create is not None: + if existing is None and ref in state.get("objectIDs", {}): + raise ContractError("managed sandbox disappeared; refuse implicit recreation", + safe_reason="SandboxStateMismatch") + if desired_state == "Stopped": + pending.append(ref) + return False + body = {**scope, **sandbox} + signature = digest(body) + if pending_create is None: + pending_create = {"generation": generation, "digest": signature} + creates[ref] = pending_create + self.store.save(state) + if (not isinstance(pending_create, dict) or pending_create.get("digest") != signature + or type(pending_create.get("generation")) is not int or pending_create["generation"] < 1): + raise ContractError("unacknowledged sandbox create changed; explicit recovery required", + safe_reason="SandboxStateMismatch") + reply = self._mutate("CreateSandbox", body, plan, pending_create["generation"]) + created = (reply or {}).get("sandbox") + if not isinstance(created, dict): + raise ContractError("sandbox creation receipt unavailable; explicit recovery required", + safe_reason="SandboxStateMismatch") + if existing and existing["metadata"]["id"] != created.get("metadata", {}).get("id"): + raise ContractError("same-name replacement detected during create replay", + safe_reason="IdentityConflict") + self._verify_sandbox(created, sandbox) + self._identity("sandbox", ws["name"], sname, created, plan, state) + del creates[ref] + self.store.save(state) + pending.append(ref) + return False + self._verify_sandbox(existing, sandbox) + self._identity("sandbox", ws["name"], sname, existing, plan, state) + phase = existing.get("status", {}).get("phase") + if phase == "SANDBOX_PHASE_ERROR": + raise ContractError("sandbox reports an error; inspect through authorized diagnostics", + safe_reason="SandboxRejected") + target = "SANDBOX_PHASE_READY" if desired_state == "Running" else "SANDBOX_PHASE_STOPPED" + if phase == target: + return True + if desired_state == "Running" and phase == "SANDBOX_PHASE_STOPPED": + self._mutate("StartSandbox", {**scope, "name": sname}, plan, generation) + elif desired_state == "Stopped" and phase == "SANDBOX_PHASE_READY": + self._mutate("StopSandbox", {**scope, "name": sname}, plan, generation) + pending.append(ref) + return False + + def _verify_sandbox(self, observed, desired): + if not self._sandbox_spec_matches(observed.get("spec", {}), desired["spec"]): + raise ContractError("sandbox configuration drift; migration is not implemented", + safe_reason="SandboxDrift") + if observed.get("metadata", {}).get("annotations", {}).get(REVISION) != desired["annotations"][REVISION]: + raise ContractError("sandbox desired revision mismatch", safe_reason="SandboxDrift") diff --git a/kubernetes-runtime/saw_kubernetes/serve.py b/kubernetes-runtime/saw_kubernetes/serve.py new file mode 100644 index 000000000..2e6e562b9 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/serve.py @@ -0,0 +1,62 @@ +"""Run one retained controller for an administrator-defined workspace.""" +import json +import os +from pathlib import Path +import signal + +from .adapters import KubernetesResourceVerifier +from .admin_session import GatewaySessions, TrustBundle +from .boundary import verify +from .compiler import compile_profile_files, ContractError +from .controller import ConfigMapClient, Controller, Scope, NAME +from .gateway_ca import read_public_ca +from .reconcile import Reconciler + + +def main(): + os.umask(0o077) + config = json.loads(Path("/config/controller.json").read_text()) + scope = Scope(config["namespace"], config["profile"], config["intent"], config["status"], config["ownerID"]) + service_account = Path("/var/run/secrets/kubernetes.io/serviceaccount") + api = ConfigMapClient("https://kubernetes.default.svc", scope, service_account / "ca.crt", service_account / "token") + def get(path): + with api._open("GET", path) as response: + return api._json(response) + + def read(kind, name, namespace=None): + if not NAME.fullmatch(name): + raise ContractError("invalid resource name") + if (kind == "runtimeclass" and namespace is None and config["mode"] == "kata" + and name == config["workloadRuntimeClass"]): + return get("/apis/node.k8s.io/v1/runtimeclasses/" + name) + if kind == "namespace" and name == scope.namespace: + return get("/api/v1/namespaces/" + name) + if kind == "persistentvolumeclaim" and namespace == scope.namespace: + return get(f"/api/v1/namespaces/{namespace}/persistentvolumeclaims/{name}") + raise ContractError("resource outside controller scope") + + def boundary(plan): + verify(config, plan, get) + resources = KubernetesResourceVerifier(read, boundary) + endpoint = f"https://{config['gateway']}-gateway.{scope.namespace}.svc:17670" + public_ca = read_public_ca(config["trustedNamespace"], config["gateway"], scope.owner_id, get) + trust = TrustBundle(api.request, scope.namespace, config["trustedNamespace"], config["gateway"], + scope.owner_id, public_ca.encode("ascii"), + os.environ["POD_NAME"], os.environ["POD_UID"]) + sessions = GatewaySessions(trust, endpoint) + state = Path("/state") + def reconcile(plan, **kwargs): + with sessions.open() as gateway: + return Reconciler(gateway, state / "core", resources.verify_attachments, + verify_shutdown=resources.verify_shutdown).reconcile(plan, **kwargs) + frontend = Controller(api, scope, state, compile_profile_files, reconcile, boundary) + stopped = False + def stop(signum, frame): + nonlocal stopped + stopped = True + signal.signal(signal.SIGTERM, stop) + frontend.run(stop=lambda: stopped) + + +if __name__ == "__main__": + main() diff --git a/kubernetes-runtime/saw_kubernetes/tool_gate_preflight.js b/kubernetes-runtime/saw_kubernetes/tool_gate_preflight.js new file mode 100644 index 000000000..c65aba0d5 --- /dev/null +++ b/kubernetes-runtime/saw_kubernetes/tool_gate_preflight.js @@ -0,0 +1,38 @@ +// The controller substitutes the digest computed by the shared VM builder. +const root = '/sandbox/.saw/trusted'; +const audit = '/sandbox/.saw/audit/tool-actions.jsonl'; +const {O_APPEND, O_WRONLY, O_NOFOLLOW} = fs.constants; +function treeFiles(dir) { + return fs.readdirSync(dir, {withFileTypes: true}).flatMap(entry => { + const path = dir + '/' + entry.name; + if (entry.isDirectory()) return treeFiles(path); + if (!entry.isFile()) throw new Error('unexpected entry in trusted gate'); + return [path]; + }); +} +function mountOptions(path) { + const entry = fs.readFileSync('/proc/self/mountinfo', 'utf8').split('\n') + .map(line => line.split(' ')).find(columns => columns[4] === path); + if (!entry) throw new Error('missing dedicated mount at ' + path); + return entry[5].split(','); +} +if (!mountOptions(root).includes('ro') || !mountOptions('/sandbox/.saw/audit').includes('rw')) + throw new Error('incorrect tool gate or audit mount mode'); +const digest = crypto.createHash('sha256'); +for (const path of treeFiles(root).sort()) { + const hash = crypto.createHash('sha256').update(fs.readFileSync(path)).digest('hex'); + digest.update(path.slice(root.length + 1) + '\0' + hash + '\n'); +} +if ('sha256:' + digest.digest('hex') !== '@SAW_GATE_DIGEST@') + throw new Error('tool gate does not match approved owner and policy'); +if (!fs.lstatSync(audit).isFile()) throw new Error('audit trail must be a regular file'); +fs.closeSync(fs.openSync(audit, O_WRONLY | O_APPEND | O_NOFOLLOW)); +try { + // No write and no truncation. A Linux append-only inode refuses this open. + fs.closeSync(fs.openSync(audit, O_WRONLY | O_NOFOLLOW)); +} catch (error) { + if (error.code !== 'EPERM') throw error; + config.plugins = {allow: ['saw-tool-gate'], load: {paths: [root + '/plugin']}, + entries: {'saw-tool-gate': {enabled: true}}}; +} +if (!config.plugins) throw new Error('audit trail permits non-append writes'); diff --git a/kubernetes-runtime/sdk-lock.json b/kubernetes-runtime/sdk-lock.json new file mode 100644 index 000000000..a08855726 --- /dev/null +++ b/kubernetes-runtime/sdk-lock.json @@ -0,0 +1,7 @@ +{ + "proto/openshell.proto": "efc7842c224157acdab9811f089e25b956ae188dc0e5a45320a5dcd43b1f2133", + "proto/datamodel.proto": "e50bd875a760f54b80269c65ee47984cae424fb623b7f59551ba62550c23e70b", + "proto/sandbox.proto": "ed3d2650c8d1c4cbb653259c6055adba2dba8a51498ab6a1b2cfc339997ab2b9", + "proto/options.proto": "10c8a99b505047f759951285fdae6068afd267ef672d00a65385a91ee8d027ee", + "tasks/scripts/generate_python_proto.py": "267078b042d0ad0a2aa09f2abfc075df8e076b7032035fa11e98f99e33d22be0" +} diff --git a/kubernetes-runtime/test-stock-protobuf.py b/kubernetes-runtime/test-stock-protobuf.py new file mode 100644 index 000000000..7ec2ebab2 --- /dev/null +++ b/kubernetes-runtime/test-stock-protobuf.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Generate the pinned stock SDK in scratch and test its real wire contracts. + +No network, cluster, package installation, source modification or image build. +Install the pinned grpcio-tools dependency separately in an isolated environment. +""" + +import argparse +import importlib.metadata +import importlib.util +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +PIN = "305c8b3d7bbdb97a6f2f7264c3ad43b0087c307f" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--openshell-source", required=True, type=Path) + parser.add_argument("--scratch-parent", type=Path) + args = parser.parse_args() + source = args.openshell_source.resolve() + revision = subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() + if revision != PIN: + parser.error(f"OpenShell source must be pinned at {PIN}") + subprocess.run(["git", "-C", str(source), "diff", "--exit-code", "HEAD", "--", + "proto", "python/openshell", "tasks/scripts/generate_python_proto.py"], + check=True, stdout=subprocess.DEVNULL) + if importlib.util.find_spec("grpc_tools") is None: + parser.error("grpcio-tools==1.80.0 is required in the isolated test environment") + if importlib.metadata.version("grpcio-tools") != "1.80.0": + parser.error("use grpcio-tools==1.80.0 for repeatable generated-SDK tests") + repo = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix="saw-stock-protobuf-", dir=args.scratch_parent) as temporary: + scratch = Path(temporary) + shutil.copytree(source / "proto", scratch / "proto") + shutil.copytree(source / "python/openshell", scratch / "python/openshell", + ignore=shutil.ignore_patterns("__pycache__", "*_test.py")) + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + subprocess.run([sys.executable, str(source / "tasks/scripts/generate_python_proto.py")], + cwd=scratch, env=env, check=True) + env["PYTHONPATH"] = os.pathsep.join([str(scratch / "python"), str(repo / "kubernetes-runtime"), + env.get("PYTHONPATH", "")]) + env["SAW_STOCK_SDK_TESTS"] = "1" + command = [sys.executable, "-c", "import pytest,sys; sys.exit(pytest.main(sys.argv[1:]))", + "-q", "-p", "no:cacheprovider", f"--basetemp={scratch / 'pytest'}", + str(repo / "tests/kubernetes_runtime")] + return subprocess.call(command, cwd=repo, env=env) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build-installer-manifest.py b/scripts/build-installer-manifest.py index e671d5aa8..a9e8acefb 100644 --- a/scripts/build-installer-manifest.py +++ b/scripts/build-installer-manifest.py @@ -16,8 +16,18 @@ 3). installer-tests.yml's drift-check step catches that: it fails if the committed bundle.sigstore.json (when one is committed) no longer verifies against a fresh render's manifest. + +The canonical saw_core.py, saw_admin.py and saw_ui_limit.py are embedded inside rendered +apply_bom.py, so their bytes are covered by that existing signed file. Keep this manifest set stable: +golden images with the original verifier can verify new bundles without an +unsigned adjacent module or a replacement verifier. Refuse raw source and +noncanonical embedded code before producing a signable payload. """ +import ast +import base64 +import binascii import hashlib +from pathlib import Path import sys import yaml @@ -25,9 +35,57 @@ FIXED_FILES = ["installer-bom.yaml", "apply_bom.py", "setup-dashboard.sh"] TOOL_GATE_FILES = ["tool-actions.yaml", "tool-gate.mjs", "tool-gate-plugin.mjs"] PKI_FILES = ["saw-gateway-pki", "openshell-gateway-pki.conf"] +CORE_SOURCE = Path(__file__).resolve().parents[1] / "charts/openshell-saw/files/installer/saw_core.py" +CORE_FIELD = "_SAW_CORE_EMBEDDED_B64" + + +def embedded_core(source, field=CORE_FIELD): + """Inspect embedded source as data; never import or execute installer code.""" + try: + tree = ast.parse(source) + except (SyntaxError, ValueError, TypeError) as exc: + raise SystemExit("rendered apply_bom.py is not valid Python") from exc + values = [statement.value for statement in tree.body + if isinstance(statement, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == field + for target in statement.targets)] + if len(values) != 1 or not isinstance(values[0], ast.Constant) or not isinstance(values[0].value, str): + raise SystemExit("rendered apply_bom.py must contain one literal shared-core embedding") + encoded = values[0].value + try: + decoded = base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error) as exc: + raise SystemExit("rendered apply_bom.py shared-core embedding is missing or malformed; render the chart first") from exc + if not decoded or base64.b64encode(decoded).decode("ascii") != encoded: + raise SystemExit("rendered apply_bom.py shared-core embedding is empty or noncanonical") + return decoded + + +def validate_shared_core(data): + if "apply_bom.py" not in data: + raise SystemExit("installer ConfigMap is missing 'apply_bom.py'; cannot build the manifest") + if "saw_core.py" in data: + raise SystemExit("installer ConfigMap must embed saw_core.py, not ship an unsigned adjacent module") + try: + canonical = CORE_SOURCE.read_bytes() + except OSError as exc: + raise SystemExit("canonical saw_core.py is missing; cannot build the manifest") from exc + if embedded_core(data["apply_bom.py"]) != canonical: + raise SystemExit("rendered apply_bom.py does not embed the canonical saw_core.py; render the chart again") + + admin_source = CORE_SOURCE.with_name("saw_admin.py") + if "saw_admin.py" in data: + raise SystemExit("installer must embed saw_admin.py inside its signed artifact") + if embedded_core(data["apply_bom.py"], "_SAW_ADMIN_EMBEDDED_B64") != admin_source.read_bytes(): + raise SystemExit("rendered apply_bom.py does not embed canonical saw_admin.py") + if "saw_ui_limit.py" in data: + raise SystemExit("installer must embed saw_ui_limit.py inside its signed artifact") + if embedded_core(data["apply_bom.py"], "_SAW_UI_LIMIT_EMBEDDED_B64") != CORE_SOURCE.with_name("saw_ui_limit.py").read_bytes(): + raise SystemExit("rendered apply_bom.py does not embed canonical saw_ui_limit.py") def build_manifest(data): + validate_shared_core(data) covered = sorted( FIXED_FILES + [name for name in data if name.startswith("provider-profile-") and name.endswith(".yaml")] diff --git a/scripts/check-rhdh-bootstrap.sh b/scripts/check-rhdh-bootstrap.sh new file mode 100755 index 000000000..b6b00ba70 --- /dev/null +++ b/scripts/check-rhdh-bootstrap.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Acceptance check around a real clean Pattern installation. +set -euo pipefail +namespace=${RHDH_NAMESPACE:-rhdh} +argo=${ARGOCD_NS:-vp-gitops} +portal=backstage-${RHDH_NAME:-developer-hub} +case "${1:-}" in + before) + existing=$(oc get secret rhdh-oidc -n "${namespace}" --ignore-not-found -o name) + if [[ -n "${existing}" ]]; then + echo "Clean bootstrap requires an installation without ${namespace}/rhdh-oidc." >&2 + exit 1 + fi + echo "Verified clean bootstrap input: ${namespace}/rhdh-oidc is absent." + ;; + after) + # ExternalSecret, hook and Application controllers reconcile asynchronously. + # The caller invokes this after the Pattern install returns. + oc wait --for=create crd/externalsecrets.external-secrets.io --timeout=15m + oc wait --for=condition=Established crd/externalsecrets.external-secrets.io --timeout=15m + for obj in externalsecret/rhdh-oidc secret/rhdh-oidc job/saw-rhdh-keycloak-client; do + oc wait --for=create "${obj}" -n "${namespace}" --timeout=15m + done + oc wait --for=condition=Ready externalsecret/rhdh-oidc -n "${namespace}" --timeout=15m + oc wait --for=condition=Complete job/saw-rhdh-keycloak-client -n "${namespace}" --timeout=15m + oc wait --for=jsonpath='{.status.sync.status}'=Synced applications.argoproj.io/openshell-rhdh -n "${argo}" --timeout=15m + oc wait --for=jsonpath='{.status.health.status}'=Healthy applications.argoproj.io/openshell-rhdh -n "${argo}" --timeout=15m + oc get applications.argoproj.io/openshell-rhdh -n "${argo}" -o json | + jq -e '.status.sync.status == "Synced" and .status.health.status == "Healthy"' >/dev/null + # The operator creates this Deployment asynchronously from the Backstage + # CR. Argo can already report Healthy while dynamic plugins are installing. + oc wait --for=create "deployment/${portal}" -n "${namespace}" --timeout=15m + oc wait --for=condition=Available "deployment/${portal}" -n "${namespace}" --timeout=30m + oc rollout status "deployment/${portal}" -n "${namespace}" --timeout=30m + oc get "deployment/${portal}" -n "${namespace}" -o json | + jq -e '.spec.replicas > 0 and .status.observedGeneration == .metadata.generation and + .status.updatedReplicas == .spec.replicas and .status.availableReplicas == .spec.replicas' >/dev/null + echo "Verified ESO Ready, Secret present, hook Complete, RHDH Synced/Healthy, and portal Deployment available." + ;; + *) echo "Usage: $0 before|after" >&2; exit 2 ;; +esac diff --git a/scripts/ci/deployment_validation.py b/scripts/ci/deployment_validation.py index 70bdaf730..4d7f83ecf 100644 --- a/scripts/ci/deployment_validation.py +++ b/scripts/ci/deployment_validation.py @@ -29,8 +29,20 @@ "openshell-gateway-docker", "governance-interceptor", "saw-credential-proxy", + "saw-kubernetes-runtime", } +CONTAINER_CHILD_CHARTS = { + "openshell-saw-kubernetes", "saw-audit-writer", "saw-kubernetes-identity", + "saw-kubernetes-proxy", "saw-runtime-policies", +} +CONTAINER_APIS = ( + "agents.x-k8s.io/v1beta1/Sandbox", + "policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy", + "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy", +) SCHEMA_LOCATIONS = ( + # The local CRD envelope uses pinned recursive definitions: the upstream + # standalone schema set omits recursive CustomResourceDefinition schemas. str(ROOT / "scripts/ci/schemas/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"), "default", "https://raw.githubusercontent.com/melmorabity/openshift-json-schemas/" @@ -88,6 +100,21 @@ def chart_args(chart: Path, mode="proxy") -> list[str]: "openshell-agents/nemoclaw-discovery-runtime@sha256:" + "0" * 64] if name == "openshell-gateway-image": return ["--set", "goldenImage.enabled=false"] + if name == "saw-image-cache": + image = yaml.safe_load((chart / "values.yaml").read_text())["holderImage"] + return ["--set", "enabled=true", "--set-string", f"images[0]={image}"] + if name == "agent-sandbox": + return ["--namespace", "agent-sandbox-system", "--set", "enabled=true"] + if name == "saw-kubernetes-controller": + return ["--namespace", "saw-ci-trusted", "-f", + str(ROOT / "tests/fixtures/saw-kubernetes-controller-values.yaml")] + if name == "saw-workspace-relay": + return ["--namespace", "saw-ci-ui", "-f", + str(ROOT / "tests/fixtures/saw-workspace-relay-values.yaml")] + if name == "openshift-home-access": + image = read_yaml(ROOT / "charts/openshell-saw/values.yaml")["bom"]["spec"]["openshell"]["sandbox"]["image"] + return ["--set-json", "images=" + json.dumps([ + {"name": "sandbox-home", "from": image, "imageUser": "1000"}])] return [] @@ -97,17 +124,37 @@ def charts() -> list[Path]: def render_charts(result: Validation, output_dir: Path) -> None: + # Validate the enabled component graph with the values actually projected by + # the composition chart, rather than maintaining a second set of child inputs. + site = read_yaml(ROOT / "tests/fixtures/container-site.yaml")["portal"]["containerSites"]["alice"] + portal_values = dict( + owner="alice", ownerSubject=site["subject"], registration="a8cb6cbd0eb62f31", + installationID="802fe583-9329-54b6-b5b4-7be58ec171f3", intentGeneration=1, + vaultPrefix="secret/data/hub/saw-alice/a8cb6cbd0eb62f31", site=site, + **{"global": {"repoURL": "https://github.com/example/secure-agent-workspace", + "targetRevision": "main", "vpArgoNamespace": "vp-gitops", + "clusterDomain": "cluster.example.test"}}) + portal_input = output_dir / "container-inputs.json" + portal_input.write_text(json.dumps(portal_values), encoding="utf-8") variants = [(path, mode) for path in charts() + if path.parent.name not in CONTAINER_CHILD_CHARTS for mode in (["proxy", "direct"] if path.parent.name == "openshell-saw" else ["proxy"])] + pending = [] for chart_file, mode in variants: chart = chart_file.parent label = chart.relative_to(ROOT).as_posix() args = chart_args(chart, mode) + if chart.name == "saw-portal-container": + args = ["--namespace", "saw-alice", "-f", str(portal_input)] if mode == "direct": label += "-direct" + pending.append((chart, chart.name, label, args)) + covered = set() + for chart, release, label, args in pending: for command in ( ["helm", "lint", str(chart), *args], - ["helm", "template", chart.name, str(chart), "--debug", *args], + ["helm", "template", release, str(chart), "--debug", *args, + *[arg for api in CONTAINER_APIS for arg in ("--api-versions", api)]], ): run = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, timeout=90, check=False) @@ -120,7 +167,26 @@ def render_charts(result: Validation, output_dir: Path) -> None: documents = list(yaml.safe_load_all(run.stdout)) if not any(isinstance(item, dict) and item.get("kind") for item in documents): result.error(label, "helm template", "no Kubernetes resources rendered") + covered.add(chart.name) result.count("charts") + if chart.name == "saw-portal-container": + for item in documents: + if not isinstance(item, dict) or item.get("kind") != "Application": + continue + source = item["spec"]["source"] + child = ROOT / source["path"] + if child.parent != ROOT / "charts" or child.name not in CONTAINER_CHILD_CHARTS: + result.error(label, "Application", f"unexpected child chart {source['path']}") + continue + helm = source["helm"] + values = helm.get("valuesObject") or yaml.safe_load(helm["values"]) + inputs = output_dir / (item["metadata"]["name"] + "-inputs.json") + inputs.write_text(json.dumps(values), encoding="utf-8") + pending.append((child, helm["releaseName"], source["path"], + ["--namespace", item["spec"]["destination"]["namespace"], + "-f", str(inputs)])) + for name in sorted(CONTAINER_CHILD_CHARTS - covered): + result.error(name, "composition", "enabled child chart was not rendered") def validate_schemas(result: Validation, output_dir: Path) -> None: @@ -128,8 +194,9 @@ def validate_schemas(result: Validation, output_dir: Path) -> None: if not rendered: result.error("charts", "schema", "no rendered resources to validate") return + # OpenShift 4.22, the minimum supported release, uses Kubernetes 1.35. command = ["kubeconform", "-strict", "-summary", "-output", "json", - "-kubernetes-version", "1.30.0"] + "-kubernetes-version", "1.35.0"] for location in SCHEMA_LOCATIONS: command.extend(("-schema-location", location)) run = subprocess.run([*command, *map(str, rendered)], cwd=ROOT, text=True, diff --git a/scripts/ci/schemas/apiextensions.k8s.io/customresourcedefinition_v1.json b/scripts/ci/schemas/apiextensions.k8s.io/customresourcedefinition_v1.json new file mode 100644 index 000000000..a321330f6 --- /dev/null +++ b/scripts/ci/schemas/apiextensions.k8s.io/customresourcedefinition_v1.json @@ -0,0 +1,44 @@ +{ + "properties": { + "apiVersion": { + "type": [ + "string", + "null" + ], + "enum": [ + "apiextensions.k8s.io/v1" + ] + }, + "kind": { + "type": [ + "string", + "null" + ], + "enum": [ + "CustomResourceDefinition" + ] + }, + "metadata": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/102361970337d65537d40141dc29b991b8e95786/v1.30.0/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" + }, + "spec": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/102361970337d65537d40141dc29b991b8e95786/v1.30.0/_definitions.json#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionSpec" + }, + "status": { + "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/102361970337d65537d40141dc29b991b8e95786/v1.30.0/_definitions.json#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinitionStatus" + } + }, + "required": [ + "spec" + ], + "type": "object", + "x-kubernetes-group-version-kind": [ + { + "group": "apiextensions.k8s.io", + "kind": "CustomResourceDefinition", + "version": "v1" + } + ], + "$schema": "http://json-schema.org/schema#", + "additionalProperties": false +} diff --git a/scripts/container-notebook-image.sh b/scripts/container-notebook-image.sh new file mode 100755 index 000000000..07d980bb7 --- /dev/null +++ b/scripts/container-notebook-image.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Publish the reviewed notebook image with a searchable /sandbox for namespace UIDs. +# stdout is a public values fragment; progress and build diagnostics go to stderr. +set -euo pipefail + +: "${BUILD_NS:?set BUILD_NS to the component build namespace}" +arch="${BUILD_ARCH:-amd64}" +case "$arch" in amd64|arm64) ;; *) echo "BUILD_ARCH must be amd64 or arm64" >&2; exit 1 ;; esac +oc get namespace "$BUILD_NS" -o name >&2 +source_image=$(uv run --locked python -c ' +import yaml +from pathlib import Path +p = Path("charts/saw-bom/profiles/data-science/default/sandbox.yaml") +print(next(s["image"] for s in yaml.safe_load(p.read_text())["spec"]["sandboxes"] if s["name"] == "notebook"))') +if [[ ! "$source_image" =~ ^[a-z0-9][a-z0-9./:_-]+@sha256:([0-9a-f]{64})$ ]]; then + echo "The notebook BOM must pin its source image by digest." >&2 + exit 1 +fi +source_digest="${BASH_REMATCH[1]}" +stream="saw-openclaw-${source_digest:0:12}-$arch" +image_user=$(oc image info --filter-by-os="linux/$arch" "$source_image" -o json | jq -er '.config.config.User | select(type=="string" and length>0)') +if [[ ! "$image_user" =~ ^[a-zA-Z0-9_-]+(:[a-zA-Z0-9_-]+)?$ || "${image_user%%:*}" == root || "${image_user%%:*}" == 0 ]]; then + echo "The notebook image must declare its non-root user." >&2 + exit 1 +fi +helm template notebook-image image-builder-charts/helm/openshift-home-access \ + --namespace "$BUILD_NS" --set configChangeTrigger=false --set-string buildArchitecture="$arch" \ + --set-string "images[0].name=$stream" --set-string "images[0].from=$source_image" \ + --set-string "images[0].imageUser=$image_user" | oc -n "$BUILD_NS" apply -f - >&2 +build=$(oc -n "$BUILD_NS" start-build "$stream" -o name) +echo "Building $build from $source_image ($arch)" >&2 +oc -n "$BUILD_NS" wait --for=jsonpath='{.status.phase}'=Complete --timeout=30m "$build" >&2 +digest=$(oc -n "$BUILD_NS" get "$build" -o json | jq -er '.status.output.to.imageDigest | select(test("^sha256:[a-f0-9]{64}$"))') +repository=$(oc -n "$BUILD_NS" get imagestream "$stream" -o json | jq -er '.status.dockerImageRepository | select(length>0)') +cat < 1 or any(v is not None and not isinstance(v, str) for v in raw): + raise ValueError("gateway and relay must use the same issuer") + if issuers: + issuer = issuers.pop() + else: + namespace = path_value(site, RELAY + ("keycloakNamespace",)) + if not isinstance(namespace, str) or not DNS_LABEL.fullmatch(namespace): + raise ValueError("expected relay Keycloak namespace") + host = command(str(SCRIPTS / "keycloak-host.sh"), namespace) + issuer = f"https://{host}/realms/{os.environ.get('KEYCLOAK_REALM', 'openshell')}" + if issuer not in checked: + checked[issuer] = discover_issuer(issuer, probe, container) + out = {} + for path in [GATEWAY + ("oidc", "issuerUrl"), RELAY + ("issuer",)]: + put(out, path, issuer) + for path in [GATEWAY + ("kubernetes", "network", "gatewayEgress"), RELAY + ("egress",)]: + put(out, path, egress(site, path, common + checked[issuer])) + put(out, BOOTSTRAP, egress(site, BOOTSTRAP, common)) + put(overlay, scope + ("containerSites", owner), out) + if not scope and sites: + put(overlay, ("projection", "egress"), egress(values, ("projection", "egress"), common)) + if not overlay: + raise ValueError("no containerSites in site values") + return overlay + + +def install_values(config, owner, overlay_file, probe, container, verify): + """Consume only managed fields from this owner's reviewed overlay.""" + site_path = ("containerSites", owner) + paths = [("projection", "egress")] + [site_path + path for path in [ + GATEWAY + ("oidc", "issuerUrl"), RELAY + ("issuer",), + GATEWAY + ("kubernetes", "network", "gatewayEgress"), RELAY + ("egress",), BOOTSTRAP]] + expected = None + if verify: + if not probe: + raise ValueError("SAW_TOPOLOGY_PROBE is required to check live container topology") + base = {"global": config.get("global", {}), "projection": config.get("projection", {}), + "containerSites": {owner: path_value(config, site_path)}} + expected = container_overlay(base, probe, container) + path_on_disk = Path(overlay_file) + if not path_on_disk.exists() and verify: + if path_value(config, site_path + ("automaticEnrollment", "enabled")) is not False: + raise ValueError("review the generated topology overlay before enabling enrollment") + # Link a complete file without overwriting another concurrent install's review. + with tempfile.NamedTemporaryFile(dir=path_on_disk.parent, mode="w") as tmp: + yaml.safe_dump(expected, tmp, sort_keys=False) + tmp.flush() + try: + os.link(tmp.name, path_on_disk) + except FileExistsError: + pass + try: + approved = yaml.safe_load(path_on_disk.read_text()) + except yaml.YAMLError: + raise ValueError("invalid YAML in container topology overlay") from None + if not isinstance(approved, dict): + raise ValueError("expected a reviewed container topology overlay") + for path in paths: + value = path_value(approved, path) + if value is None or (verify and value != path_value(expected, path)): + raise ValueError("missing or stale container topology at " + ".".join(path) + + "; regenerate from base inputs and review the changes") + for path in paths: + put(config, path, path_value(approved, path)) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--values-file", default=os.environ.get("SAW_USERS_VALUES_FILE"), type=Path) + p.add_argument("--probe", default=os.environ.get("SAW_TOPOLOGY_PROBE"), help="trusted namespace/pod with Python 3 and issuer CA trust") + p.add_argument("--container", default=os.environ.get("SAW_TOPOLOGY_CONTAINER")) + p.add_argument("--check", type=Path, help="compare a reviewed overlay with live observations") + args = p.parse_args() + if not args.values_file or not args.probe: + p.error("supply the base values file and trusted probe namespace/pod") + try: + found = container_overlay(yaml.safe_load(args.values_file.read_text()), args.probe, args.container) + if args.check: + if yaml.safe_load(args.check.read_text()) != found: + raise ValueError("container topology drift; regenerate the overlay from the base file and review the diff") + else: + sys.stdout.write(yaml.safe_dump(found, sort_keys=False)) + except yaml.YAMLError: + p.exit(1, "Container topology check failed: invalid YAML input\n") + except (ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError) as exc: + p.exit(1, f"Container topology check failed: {exc}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy-keycloak.sh b/scripts/deploy-keycloak.sh index 01fa8c17f..e64a37f82 100755 --- a/scripts/deploy-keycloak.sh +++ b/scripts/deploy-keycloak.sh @@ -17,6 +17,11 @@ DEMO_ENABLED="${DEMO_ENABLED:-false}" exit 2 } +wait_for_realm_import() { + oc wait -n "${NS}" --for=condition=Done \ + keycloakrealmimport/openshell-keycloak-realm --timeout=5m +} + # Check RHBK operator access and location before any writes. csv_json="$(oc get csv -n "${NS}" -o json)" if ! jq -e '.items | any(.metadata.name | contains("rhbk"))' \ @@ -80,13 +85,7 @@ if [[ -n "${existing}" && -z "${ours}" ]]; then helm upgrade --install openshell-keycloak "${CHART}" --namespace "${NS}" \ --set keycloak.existing="${existing}" --set keycloak.realm="${REALM}" \ --set demo.enabled="${DEMO_ENABLED}" --timeout 10m - deadline=$((SECONDS + 300)) - until [[ "$(oc get keycloakrealmimport -n "${NS}" -o json | jq -r \ - 'first(.items[] | select(.metadata.name == "openshell-keycloak-realm") | - .status.conditions[]? | select(.type == "Done") | .status) // empty')" == "True" ]]; do - if (( SECONDS > deadline )); then echo "Timed out waiting for the realm import." >&2; exit 1; fi - echo " waiting for the realm import..."; sleep 10 - done + wait_for_realm_import KEYCLOAK_NS="${NS}" KEYCLOAK_REALM="${REALM}" "${SCRIPT_DIR}/keycloak-check.sh" exit 0 fi @@ -106,7 +105,8 @@ while true; do .status.conditions[]? | select(.type == "Ready") | .status) // empty' \ <<<"${keycloaks}")" if [[ "${ready}" == "True" ]]; then - echo "Keycloak is ready." + wait_for_realm_import + echo "Keycloak and its realm import are ready." KC_URL="$(jq -r 'first(.items[] | select(.metadata.name == "openshell-keycloak") | .status.externalURL) // empty' <<<"${keycloaks}")" if [[ -n "${KC_URL}" ]]; then diff --git a/scripts/kata_worker_check.py b/scripts/kata_worker_check.py new file mode 100644 index 000000000..9447f8314 --- /dev/null +++ b/scripts/kata_worker_check.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Check KVM device access on selected x86 local-Kata workers.""" +import json +import os +import subprocess +import sys + + +def oc(*args): + return subprocess.check_output(["oc", "--request-timeout=30s", *args], text=True) + + +def workers(items): + if not items: + raise ValueError("worker selector matched no nodes") + for node in items: + meta = node["metadata"] + labels = meta["labels"] + roles = {k.removeprefix("node-role.kubernetes.io/") for k in labels + if k.startswith("node-role.kubernetes.io/")} + if "worker" not in roles or roles & {"master", "control-plane"}: + raise ValueError(f"{meta['name']}: selector must match dedicated workers only") + if labels.get("kubernetes.io/arch") != "amd64": + raise ValueError(f"{meta['name']}: this preflight qualifies x86_64 workers only") + if node["spec"].get("unschedulable") or not any( + c["type"] == "Ready" and c["status"] == "True" for c in node["status"]["conditions"] + ): + raise ValueError(f"{meta['name']}: worker is unavailable") + return sorted(n["metadata"]["name"] for n in items) + + +# Inspect the host through the existing machine-config-daemon host mount. +# Reading an inaccessible/missing mount must not count as a firmware verdict. +PROBE = """set -eu +test -r /proc/sys/kernel/random/boot_id && test -d /sys/firmware || exit 1 +if test -d /sys/firmware/efi; then echo UEFI; else echo legacy-BIOS; fi +if test -c /dev/kvm; then echo KVM; else echo missing-KVM; fi +""" + + +def check(expected, selector): + selector = (selector or "").strip() + expected = (expected or "").strip() + if not selector or not expected: + raise ValueError("set KATA_API_SERVER and KATA_NODE_SELECTOR") + if oc("whoami", "--show-server").strip() != expected: + raise ValueError("unexpected OpenShift API server") + names = workers(json.loads(oc("get", "nodes", "--selector", selector, "-o", "json"))["items"]) + ns = "openshift-machine-config-operator" + passed = True + for name in names: + pods = json.loads(oc("-n", ns, "get", "pods", "-l", "k8s-app=machine-config-daemon", + "--field-selector", f"spec.nodeName={name},status.phase=Running", + "-o", "json"))["items"] + ready = [p["metadata"]["name"] for p in pods if not p["metadata"].get("deletionTimestamp") + and any(c["name"] == "machine-config-daemon" and c["ready"] + for c in p["status"].get("containerStatuses", []))] + if len(ready) != 1: + raise ValueError(f"{name}: cannot identify a ready machine-config-daemon") + facts = oc("-n", ns, "exec", ready[0], "-c", "machine-config-daemon", "--", + "chroot", "/rootfs", "sh", "-c", PROBE).splitlines() + print(f"{name}: {' '.join(facts)}") + passed &= len(facts) == 2 and facts[0] in {"UEFI", "legacy-BIOS"} and facts[1] == "KVM" + if not passed: + raise ValueError("selected workers must expose /dev/kvm; firmware is reported for context only") + return names + + +def main(): + selector = os.environ.get("KATA_NODE_SELECTOR", "").strip() + expected = os.environ.get("KATA_API_SERVER", "").strip() + check(expected, selector) + + +if __name__ == "__main__": + try: + main() + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as exc: + sys.exit(f"Kata worker preflight failed: {exc}") diff --git a/scripts/oidc_topology_probe.py b/scripts/oidc_topology_probe.py new file mode 100644 index 000000000..f909a97d8 --- /dev/null +++ b/scripts/oidc_topology_probe.py @@ -0,0 +1,47 @@ +"""Verify OIDC discovery from the trusted Pod's DNS and TLS trust context.""" +import http.client +import ipaddress +import json +import socket +import ssl +import sys +from urllib.parse import urlsplit + + +def probe(issuer): + url = urlsplit(issuer) + port = url.port or 443 + answers = {row[4][0] for row in socket.getaddrinfo(url.hostname, port, type=socket.SOCK_STREAM)} + context = ssl.create_default_context() + for address in sorted(answers): + ip = ipaddress.ip_address(address) + if ip.version != 4 or ip.is_unspecified or ip.is_loopback or ip.is_multicast or ip.is_link_local or ip.is_reserved: + raise ValueError("unsupported or unsafe issuer address") + # Connect directly to each DNS answer. SNI, hostname verification and + # the HTTP Host header still use the configured issuer's hostname. + conn = http.client.HTTPSConnection(url.hostname, port, timeout=5, context=context) + try: + with socket.create_connection((address, port), timeout=5) as sock: + conn.sock = context.wrap_socket(sock, server_hostname=url.hostname) + conn.request("GET", url.path.rstrip("/") + "/.well-known/openid-configuration") + response = conn.getresponse() + data = response.read(1024 * 1024 + 1) + if response.status != 200 or len(data) > 1024 * 1024 or json.loads(data).get("issuer") != issuer: + raise ValueError("issuer discovery mismatch") + finally: + conn.close() + return sorted(answers) + + +if __name__ == "__main__": + try: + result = probe(sys.argv[1]) + except socket.gaierror: + result = {"error": "dns"} + except ssl.SSLError: + result = {"error": "tls"} + except (OSError, http.client.HTTPException): + result = {"error": "connection"} + except (ValueError, AttributeError): + result = {"error": "discovery"} + print(json.dumps(result)) diff --git a/scripts/openshell-saw-create.sh b/scripts/openshell-saw-create.sh index e2e9372ce..0b60f0492 100755 --- a/scripts/openshell-saw-create.sh +++ b/scripts/openshell-saw-create.sh @@ -111,7 +111,10 @@ if [[ -n "${SAW_VALUES_FILE}" ]]; then fi additional_provider_secrets=() if [[ -n "${SAW_VALUES_FILE}" ]]; then - mapfile -t additional_provider_secrets < <(python3 -c 'import sys,yaml; d=yaml.safe_load(open(sys.argv[1])) or {}; print("\n".join(d.get("additionalProviderSecrets") or []))' "${SAW_VALUES_FILE}") + provider_names="$(python3 -c 'import sys,yaml; d=yaml.safe_load(open(sys.argv[1])) or {}; print("\n".join(d.get("additionalProviderSecrets") or []))' "${SAW_VALUES_FILE}")" + while IFS= read -r name; do + [[ -z "$name" ]] || additional_provider_secrets+=("$name") + done <<< "$provider_names" fi python3 - "${SAW_VALUES_FILE}" "${GPU_ENABLED_OVERRIDE}" "${GPU_ENABLED}" \ "${GPU_COUNT_OVERRIDE}" "${GPU_COUNT}" "${GPU_DEVICE_NAME_OVERRIDE}" "${GPU_DEVICE_NAME}" <<'PY' @@ -374,7 +377,7 @@ if [[ "${CREDENTIAL_PROXY_ENABLED}" == true ]]; then if [[ -n "$(oc get secret inference -n "${DEPLOY_NS}" -o name --ignore-not-found)" ]]; then sync_provider_metadata inference fi -for name in "${additional_provider_secrets[@]}"; do +for name in ${additional_provider_secrets[@]+"${additional_provider_secrets[@]}"}; do [[ -n "${name}" && "${name}" != web-search ]] || continue if [[ -n "$(oc get secret "${name}" -n "${DEPLOY_NS}" -o name --ignore-not-found)" ]]; then sync_provider_metadata "${name}" @@ -408,7 +411,7 @@ if ! grep -qx 'secret/inference' <<<"${secret_names}"; then SECRET_SET+=(--set "inference.secretName=") fi if ! grep -qx 'secret/web-search' <<<"${secret_names}"; then - extra_secrets="$(printf '%s\n' "${additional_provider_secrets[@]}" | jq -Rsc 'split("\n") | map(select(length > 0 and . != "web-search"))')" + extra_secrets="$(printf '%s\n' ${additional_provider_secrets[@]+"${additional_provider_secrets[@]}"} | jq -Rsc 'split("\n") | map(select(length > 0 and . != "web-search"))')" SECRET_SET+=(--set-json "additionalProviderSecrets=${extra_secrets}") web_search_enabled=false fi @@ -495,7 +498,7 @@ fi if grep -qx 'secret/web-search' <<<"${secret_names}"; then proxy_bindings="$(jq -c 'if any(.[]; .name == "web-search" and .key == "api_key") then . else . + [{name:"web-search",key:"api_key",providers:["brave"]}] end' <<<"${proxy_bindings}")" fi -for name in "${additional_provider_secrets[@]}"; do +for name in ${additional_provider_secrets[@]+"${additional_provider_secrets[@]}"}; do [[ -n "${name}" && "${name}" != web-search ]] || continue if grep -qx "secret/${name}" <<<"${secret_names}"; then if jq -e --arg name "${name}" 'any(.[]; .name == $name)' <<<"${proxy_bindings}" >/dev/null; then continue; fi @@ -514,7 +517,7 @@ if [[ "${CREDENTIAL_PROXY_ENABLED}" == false ]]; then while IFS= read -r name; do [[ -z "${name}" ]] || stale_meta+=("${name}-meta") done < <(jq -r '.data["config.json"] // "{}" | fromjson | .credentialProxy.secrets[]?.name' <<<"${previous_installer}") - for name in "${additional_provider_secrets[@]}"; do + for name in ${additional_provider_secrets[@]+"${additional_provider_secrets[@]}"}; do [[ -z "${name}" ]] || stale_meta+=("${name}-meta") done oc delete secret "${stale_meta[@]}" saw-credential-proxy-ca -n "${DEPLOY_NS}" --ignore-not-found diff --git a/scripts/pattern-drain-applications.py b/scripts/pattern-drain-applications.py new file mode 100644 index 000000000..7ad194fe5 --- /dev/null +++ b/scripts/pattern-drain-applications.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Remove SAW consumers through Argo before the framework removes their webhooks.""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +PATTERN = "secure-agent-workspace" +ARGO_NS = os.environ.get("ARGOCD_NS", "vp-gitops") +PATTERN_NS = "patterns-operator" +# Keys in values-prod.yaml. Both leaf charts create webhook-dependent resources. +CHARTS = {"openshell-rhdh": "charts/openshell-rhdh", "saw-users": "charts/saw-users"} +GROUP = "applications.argoproj.io" +PATTERNS = "patterns.gitops.hybrid-cloud-patterns.io" + + +def oc(*args, body=None): + result = subprocess.run(["oc", "--request-timeout=20s", *args], + input=json.dumps(body) if body is not None else None, + text=True, capture_output=True) + if result.returncode: + raise RuntimeError(result.stderr.strip() or "OpenShift API request failed") + return result.stdout + + +def get(kind, name, namespace): + raw = oc("get", kind, name, "-n", namespace, "--ignore-not-found", "-o", "json") + return json.loads(raw) if raw.strip() else None + + +def metadata(obj): + return obj["metadata"] + + +def patch(kind, obj, edits): + m = metadata(obj) + checks = [{"op": "test", "path": "/metadata/uid", "value": m["uid"]}] + # The Pattern operator writes status between discovery and this request. + # Compare the complete spec used to compute the overrides. Argo operations + # below also depend on status and therefore require the resource version. + if kind == PATTERNS: + checks.append({"op": "test", "path": "/spec", "value": obj["spec"]}) + else: + checks.append({"op": "test", "path": "/metadata/resourceVersion", "value": m["resourceVersion"]}) + oc("patch", kind, m["name"], "-n", m["namespace"], "--type=json", + "--patch-file=/dev/stdin", body=checks + edits) + + +def local_destination(app): + destination = app["spec"].get("destination", {}) + return ((destination.get("name"), destination.get("server")) + in (("in-cluster", None), (None, "https://kubernetes.default.svc"))) + + +def validate_child(child, parent, name, uid): + m = metadata(child) + tracking = f"{parent}:argoproj.io/Application:{ARGO_NS}/{name}" + if (m["uid"] != uid or m["name"] != name or m["namespace"] != ARGO_NS + or not local_destination(child) + or (not m.get("deletionTimestamp") and + "resources-finalizer.argocd.argoproj.io/foreground" not in m.get("finalizers", [])) + or m.get("labels", {}).get("validatedpatterns.io/pattern") != PATTERN + or m.get("annotations", {}).get("argocd.argoproj.io/tracking-id") != tracking + or child["spec"].get("source", {}).get("path") != CHARTS[name]): + raise RuntimeError(f"Refusing to drain unverified or replaced Application {name}") + + +def disable_consumers(pattern, names): + # Persist the desired change on the Pattern itself. Patching the child or + # its generated parent alone would let a reconciler recreate it mid-delete. + params = pattern["spec"].get("extraParameters", []) + edits = [] if "extraParameters" in pattern["spec"] else [ + {"op": "add", "path": "/spec/extraParameters", "value": []}] + for name in names: + key = f"clusterGroup.applications.{name}.disabled" + matches = [i for i, param in enumerate(params) if param["name"] == key] + if len(matches) > 1: + raise RuntimeError(f"Ambiguous Pattern override for {name}") + if not matches: + edits.append({"op": "add", "path": "/spec/extraParameters/-", + "value": {"name": key, "value": "true"}}) + elif params[matches[0]]["value"] != "true": + edits.append({"op": "replace", "path": f"/spec/extraParameters/{matches[0]}/value", + "value": "true"}) + if edits: + patch(PATTERNS, pattern, edits) + + +def released(parent, name): + return any(r.get("group") == "argoproj.io" and r.get("kind") == "Application" + and r.get("namespace") == ARGO_NS and r.get("name") == name + and r.get("requiresPruning") is True + for r in parent.get("status", {}).get("resources", [])) + + +def drain_child(child): + m = metadata(child) + if child["spec"].get("syncPolicy", {}).get("automated") is not None: + patch(GROUP, child, [{"op": "remove", "path": "/spec/syncPolicy/automated"}]) + elif child.get("operation") is not None: + phase = child.get("status", {}).get("operationState", {}).get("phase") + if phase == "Running": + # Same transition as Argo's TerminateOperation; the controller + # terminates the hooks and records the result itself. + patch(GROUP, child, [{"op": "replace", "path": "/status/operationState/phase", + "value": "Terminating"}]) + elif not m.get("deletionTimestamp"): + path = f"/apis/argoproj.io/v1alpha1/namespaces/{ARGO_NS}/applications/{m['name']}" + oc("delete", "--raw", path, "-f", "-", body={ + "apiVersion": "v1", "kind": "DeleteOptions", "propagationPolicy": "Foreground", + "preconditions": {"uid": m["uid"], "resourceVersion": m["resourceVersion"]}}) + + +def main(): + exists = subprocess.run([str(Path(__file__).with_name("pattern-exists.sh"))]) + if exists.returncode == 1: + return + exists.check_returncode() + pattern = get(PATTERNS, PATTERN, PATTERN_NS) + if not pattern: + raise RuntimeError("Pattern identity could not be verified in patterns-operator") + parent_name = PATTERN + "-" + pattern["spec"].get("clusterGroupName", "hub") + ids = {} + for name in CHARTS: + child = get(GROUP, name, ARGO_NS) + if child: + uid = metadata(child)["uid"] + validate_child(child, parent_name, name, uid) + ids[name] = uid + if not ids: + return + parent = get(GROUP, parent_name, ARGO_NS) + if (not parent or not local_destination(parent) + or metadata(parent).get("labels", {}).get("validatedpatterns.io/pattern") != PATTERN): + raise RuntimeError("Pattern parent Application ownership could not be verified") + parent_uid = metadata(parent)["uid"] + if not metadata(pattern).get("deletionTimestamp"): + disable_consumers(pattern, ids) + print("Draining SAW Applications before uninstall: " + ", ".join(ids), flush=True) + deadline = time.monotonic() + 900 + while ids and time.monotonic() < deadline: + parent = get(GROUP, parent_name, ARGO_NS) + if parent and metadata(parent)["uid"] != parent_uid: + raise RuntimeError("Pattern parent Application was replaced during uninstall") + for name, uid in list(ids.items()): + child = get(GROUP, name, ARGO_NS) + if not child: + del ids[name] + continue + validate_child(child, parent_name, name, uid) + if not metadata(child).get("deletionTimestamp") and not (parent and released(parent, name)): + continue + try: + drain_child(child) + except RuntimeError: + # Retry a concurrent update only on the same object. Other + # failures, including authorization errors, stop uninstall. + current = get(GROUP, name, ARGO_NS) + if current and (metadata(current)["uid"] != uid or + metadata(current)["resourceVersion"] == metadata(child)["resourceVersion"]): + raise + if ids: + time.sleep(4) + if ids: + raise RuntimeError("Timed out draining " + ", ".join(ids) + "; inspect the parent and leaf Applications") + + +if __name__ == "__main__": + try: + main() + except (OSError, RuntimeError, subprocess.CalledProcessError) as exc: + sys.exit(str(exc)) diff --git a/scripts/pattern-operator-cleanup.sh b/scripts/pattern-operator-cleanup.sh index cc96a546b..fc22a5509 100755 --- a/scripts/pattern-operator-cleanup.sh +++ b/scripts/pattern-operator-cleanup.sh @@ -2,6 +2,13 @@ # Run only after the framework removes the Argo CD applications. set -euo pipefail +hco_storage_resource() { + # The preferred API version can require a conversion webhook from the + # operator being uninstalled. Address the served storage version directly. + jq -er '[.spec.versions[] | select(.served and .storage)] | + select(length == 1) | "hyperconvergeds." + .[0].name + ".hco.kubevirt.io"' +} + namespaces="$(oc get namespaces -o json)" pattern_namespace="$(jq -r '.items[] | select(.metadata.name == "openshift-cnv") | @@ -22,7 +29,11 @@ if [[ "${namespace_present}" == true ]]; then clusterserviceversions.operators.coreos.com \ installplans.operators.coreos.com; do if ! grep -qx "${kind}" <<<"${resources}"; then continue; fi - objects="$(oc get "${kind}" -n openshift-cnv -o json)" + resource="${kind}" + if [[ "${kind}" == hyperconvergeds.hco.kubevirt.io ]]; then + resource="$(oc get crd "${kind}" -o json | hco_storage_resource)" + fi + objects="$(oc get "${resource}" -n openshift-cnv -o json)" owned_names="$(jq -r '.items[] | select(.metadata.labels["argocd.argoproj.io/instance"] == "openshift-cnv" or ((.metadata.annotations["argocd.argoproj.io/tracking-id"] // "") | @@ -38,7 +49,7 @@ if [[ "${namespace_present}" == true ]]; then while IFS= read -r name; do [[ -n "${name}" ]] || continue echo "Deleting pattern-owned ${kind}/${name} in openshift-cnv." - oc delete "${kind}" "${name}" -n openshift-cnv \ + oc delete "${resource}" "${name}" -n openshift-cnv \ --wait=false --ignore-not-found=true done <<<"${owned_names}" done @@ -54,10 +65,14 @@ if [[ "${namespace_present}" == true && \ fi if [[ "${namespace_present}" == true ]]; then - if ! oc wait --for=delete namespace/openshift-cnv --timeout=180s; then - if [[ -n "$(oc get namespace openshift-cnv -o name --ignore-not-found)" ]]; then - echo "Warning: openshift-cnv still exists after 180 s; keeping its HyperConverged CRD." >&2 - exit 0 + # OLM and namespace event collection can outlast the operator workloads. + # Use the same one-hour bound as the framework uninstall. + if ! oc wait --for=delete namespace/openshift-cnv --timeout=1h; then + remaining="$(oc get namespace openshift-cnv -o name --ignore-not-found)" + if [[ -n "${remaining}" ]]; then + echo "CNV teardown incomplete: openshift-cnv still exists; its HyperConverged CRD is retained." >&2 + echo "After the namespace terminates, rerun make pattern-uninstall before reinstalling." >&2 + exit 1 fi fi fi @@ -75,7 +90,8 @@ if ! jq -e '.metadata.annotations["openshell.pattern/cleanup-on-uninstall"] == exit 0 fi -hyperconvergeds="$(oc get hyperconvergeds.hco.kubevirt.io -A -o json)" +hco_resource="$(hco_storage_resource <<<"${crd}")" +hyperconvergeds="$(oc get "${hco_resource}" -A -o json)" subscriptions="$(oc get subscriptions.operators.coreos.com -A -o json)" if jq -e '.items | length > 0' <<<"${hyperconvergeds}" >/dev/null || jq -e '.items | any(.spec.name == "kubevirt-hyperconverged")' \ diff --git a/scripts/portal-container/check-pod-admission.py b/scripts/portal-container/check-pod-admission.py new file mode 100644 index 000000000..6fb01cbcd --- /dev/null +++ b/scripts/portal-container/check-pod-admission.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Exercise the installed SAW Pod policy with server-side dry-runs.""" +import argparse +import copy +import json +import subprocess + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for option in ("kubeconfig", "namespace", "gateway"): + parser.add_argument("--" + option, required=True) + parser.add_argument("--fixture-namespace", help="Optional source of matching stock PodSpecs for an isolated policy test") + parser.add_argument("--sandbox-controller", default="system:serviceaccount:agent-sandbox-system:agent-sandbox-controller", + help="Agent Sandbox controller identity configured on the site") + args = parser.parse_args() + oc = ["oc", "--kubeconfig", args.kubeconfig, "-n", args.namespace, "--request-timeout=20s"] + account = f"system:serviceaccount:{args.namespace}:{args.gateway}-gateway" + target = json.loads(subprocess.check_output(oc + ["get", "pods", "-o", "json"]))["items"] + if args.fixture_namespace: + items = json.loads(subprocess.check_output(oc + ["-n", args.fixture_namespace, "get", "pods", "-o", "json"]))["items"] + else: + items = target + + def fixture(role): + pod = next(p for p in items if p["metadata"].get("labels", {}).get("openshell.ai/boundary-role") == role) + meta = {k: copy.deepcopy(pod["metadata"][k]) for k in ("name", "labels", "ownerReferences")} + # A copied owner reference cannot confer permission to block owner deletion. + for owner in meta["ownerReferences"]: + owner.pop("blockOwnerDeletion", None) + spec = copy.deepcopy(pod["spec"]) + spec.pop("nodeName", None) + if args.fixture_namespace and args.fixture_namespace != args.namespace: + spec.pop("imagePullSecrets", None) + for container in [spec, *spec["containers"], *spec.get("initContainers", [])]: + for field in ("runAsUser", "runAsGroup", "fsGroup", "seLinuxOptions", "supplementalGroups"): + container.get("securityContext", {}).pop(field, None) + return {"apiVersion": "v1", "kind": "Pod", "metadata": meta, "spec": spec} + + def denied(label, command, *, obj=None, identity=account): + caller = ["--as=" + identity, "--as-group=system:authenticated", "--as-group=system:serviceaccounts", + "--as-group=system:serviceaccounts:" + identity.split(":")[2]] + result = subprocess.run(oc + caller + command + ["--dry-run=server"], text=True, + input=json.dumps(obj) if obj else None, capture_output=True) + policy = ("saw-" + args.namespace + "-runtime")[:63].rstrip("-") + if result.returncode == 0 or "ValidatingAdmissionPolicy" not in result.stderr or policy not in result.stderr: + raise SystemExit(f"{label}: expected denial from {policy}: {result.stderr.strip()}") + print(label + ": denied by the workspace Pod policy") + + workload = fixture("workload") + denied("forged workload creator", ["create", "-f", "-"], obj=workload) + workload["spec"]["initContainers"][0]["env"] = [{ + "name": "REVIEW", "valueFrom": {"secretKeyRef": {"name": "nonexistent-review-fixture", "key": "review"}}}] + denied("unapproved workload generated by the Sandbox controller", ["create", "-f", "-"], + obj=workload, identity=args.sandbox_controller) + pod = fixture("supervisor") + pod["spec"]["volumes"].append({"name": "review", "secret": {"secretName": "nonexistent-review-fixture"}}) + denied("unapproved supervisor Secret", ["create", "-f", "-"], obj=pod) + unmarked = fixture("workload") + unmarked["metadata"] = {"name": "saw-admission-negative-check"} + unmarked["spec"]["serviceAccountName"] = "default" + denied("unmarked Pod", ["create", "-f", "-"], obj=unmarked) + patch = [{"op": "replace", "path": "/spec/containers/0/image", "value": "registry.k8s.io/pause:3.10"}] + running = next(p for p in target if p["metadata"].get("labels", {}).get("openshell.ai/boundary-role") == "workload") + role_patch = [{"op": "remove", "path": "/metadata/labels/openshell.ai~1boundary-role"}] + denied("removal of workload identity", ["patch", "pod", running["metadata"]["name"], "--type=json", + "-p", json.dumps(role_patch)]) + denied("workload image replacement", ["patch", "pod", running["metadata"]["name"], "--type=json", + "-p", json.dumps(patch)]) + + +if __name__ == "__main__": + main() diff --git a/scripts/portal-container/inspect-boundary.py b/scripts/portal-container/inspect-boundary.py new file mode 100644 index 000000000..9e04d462f --- /dev/null +++ b/scripts/portal-container/inspect-boundary.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Read the public objects that form a site approval. No Kubernetes writes or Secret reads.""" +import argparse +from functools import lru_cache +import json +from pathlib import Path +import re +import subprocess +import sys +from uuid import UUID, uuid5, NAMESPACE_URL + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "kubernetes-runtime")) +from saw_kubernetes.audit import capture +from saw_kubernetes.boundary import fingerprint, objects +from saw_kubernetes.portal_contract import contract + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("owner", "registration", "governance-namespace", "api-server"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--installation-id", help="Explicit installation UUID for static values; defaults to the portal registration identity") + output = parser.add_mutually_exclusive_group() + output.add_argument("--show-objects", action="store_true", help="print the underlying public specs/data for review") + output.add_argument("--profile-digest", action="store_true", help="print the rendered public BOM data digest for review") + parser.add_argument("--kata-runtime-class", help="review the Kata workload boundary for this class") + a = parser.parse_args() + if not re.fullmatch("[a-z0-9]([-a-z0-9]{0,17}[a-z0-9])?", a.owner) or not re.fullmatch("[a-f0-9]{16}", a.registration): + parser.error("invalid owner or registration") + if not re.fullmatch("[a-z0-9]([-a-z0-9]*[a-z0-9])?", a.governance_namespace): + parser.error("invalid governance namespace") + if subprocess.check_output(["oc", "whoami", "--show-server"], text=True).strip() != a.api_server: + parser.error("unexpected cluster") + + @lru_cache(maxsize=None) + def read(path): + return json.loads(subprocess.check_output(["oc", "get", "--raw", path, "--request-timeout=20s"])) + + ns = "saw-" + a.owner + meta = read("/api/v1/namespaces/" + ns)["metadata"] + ident = a.installation_id or str(uuid5(NAMESPACE_URL, "urn:saw:portal:" + a.owner + ":" + a.registration)) + try: + if str(UUID(ident)) != ident or UUID(ident).int == 0: + raise ValueError() + except ValueError: + parser.error("invalid installation identity") + if meta.get("deletionTimestamp") or meta.get("labels", {}).get("saw.redhat.com/runtime-owner") != ident: + parser.error("runtime namespace does not belong to this live registration") + config = dict(namespace=ns, namespaceUID=meta["uid"], trustedNamespace="saw-ctl-" + a.registration, + ownerID=ident, mode="container", governanceNamespace=a.governance_namespace, + gateway=a.owner, proxyPolicy=a.owner + "-proxy-policy") + if a.kata_runtime_class: + if not re.fullmatch("[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?", a.kata_runtime_class): + parser.error("invalid RuntimeClass") + config.update(mode="kata", workloadRuntimeClass=a.kata_runtime_class) + capture(config, read) # same health, admission and network checks as enrollment + if a.profile_digest: + print(fingerprint(read(f"/api/v1/namespaces/{ns}/configmaps/saw-bom-profiles")["data"])) + return + from saw_kubernetes.core import shared + files = read(f"/api/v1/namespaces/{ns}/configmaps/saw-bom-profiles")["data"] + documents, _, _ = shared.split_bom_files(files) + config["slotKeys"] = sorted(ws.name+"/"+sb.name + for _, ws in shared.enabled_workspaces(shared.parse_profiles(documents)) + for sb in ws.sandboxes if sb.enabled) + reference = contract(config, read) + result = ({key: {"path":path, field:read(path)[field]} for key,(path,field) in objects(config).items()} + if a.show_objects else reference) + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/portal-container/setup-relay-provisioner.py b/scripts/portal-container/setup-relay-provisioner.py new file mode 100644 index 000000000..9517e6b75 --- /dev/null +++ b/scripts/portal-container/setup-relay-provisioner.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Render or run the one-time, platform-owned Keycloak relay registrar setup.""" +import argparse +import base64 +import json +import os +from pathlib import Path +import secrets +import ssl +import sys +from urllib.error import HTTPError +from urllib.parse import urlencode, urlsplit +from urllib.request import build_opener, HTTPRedirectHandler, HTTPSHandler, Request + +NAME = "saw-relay-provisioner" +SCOPE = "saw-workspace-relay" +IMAGE = "registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c" + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args): + return None + + +def call(url, method="GET", *, token=None, body=None, form=None, ca=None, patch=False): + headers = {} + data = None + if form is not None: + headers["Content-Type"] = "application/x-www-form-urlencoded" + data = urlencode(form).encode() + if body is not None: + headers["Content-Type"] = "application/merge-patch+json" if patch else "application/json" + data = json.dumps(body).encode() + if token: + headers["Authorization"] = "Bearer " + token + client = build_opener(NoRedirect(), HTTPSHandler(context=ssl.create_default_context(cafile=ca))) + with client.open(Request(url, data=data, method=method, headers=headers), timeout=20) as response: + raw = response.read() + return json.loads(raw) if raw else None + + +def setup(): + issuer = os.environ["ISSUER"].rstrip("/") + base, realm = issuer.rsplit("/realms/", 1) + if urlsplit(issuer).scheme != "https" or not realm or realm == "master": + raise ValueError("expected a dedicated HTTPS realm") + admin = Path("/bootstrap") + token = call(base + "/realms/master/protocol/openid-connect/token", "POST", + form={"client_id":"admin-cli", "grant_type":"password", + "username":(admin/"username").read_text().strip(), + "password":(admin/"password").read_text().strip()})["access_token"] + def kc(path, method="GET", body=None): + return call(base + "/admin/realms/" + realm + path, method, token=token, body=body) + + sa = Path("/var/run/secrets/kubernetes.io/serviceaccount") + secret_url = ("https://kubernetes.default.svc/api/v1/namespaces/" + os.environ["NAMESPACE"] + + "/secrets/" + NAME) + opts = dict(ca=str(sa/"ca.crt"), token=(sa/"token").read_text().strip()) + state = call(secret_url, **opts) + if state["metadata"]["labels"].get("saw.redhat.com/component") != NAME: + raise ValueError("unexpected destination Secret") + if not state.get("data"): + state = call(secret_url, "PATCH", patch=True, **opts, body={ + "metadata":{"resourceVersion":state["metadata"]["resourceVersion"]}, + "stringData":{"client-id":NAME, "client-secret":secrets.token_urlsafe(40), "issuer":issuer}}) + values = {k:base64.b64decode(v,validate=True).decode() for k,v in state["data"].items()} + if set(values) != {"issuer","client-id","client-secret"} or values["issuer"] != issuer or values["client-id"] != NAME: + raise ValueError("incompatible provisioner Secret") + audience = os.environ["AUDIENCE"] + mappers = [ + {"name":"relay-audience", "protocol":"openid-connect", "protocolMapper":"oidc-audience-mapper", + "config":{"included.client.audience":audience, "access.token.claim":"true"}}, + {"name":"relay-user", "protocol":"openid-connect", "protocolMapper":"oidc-hardcoded-claim-mapper", + "config":{"claim.name":"realm_access.roles","claim.value":'["openshell-user"]', + "jsonType.label":"JSON","access.token.claim":"true"}}] + scopes = {s["name"]:s for s in kc("/client-scopes")} + if SCOPE not in scopes: + kc("/client-scopes","POST",{"name":SCOPE, "protocol":"openid-connect", + "description":"SAW registration-scoped workspace relay", "protocolMappers":mappers}) + scope = next(s for s in kc("/client-scopes") if s["name"]==SCOPE) + actual = kc("/client-scopes/"+scope["id"]+"/protocol-mappers/models") + if {m["name"] for m in actual} != {m["name"] for m in mappers}: + raise ValueError("unexpected relay scope mappers") + for desired in mappers: + found=next(m for m in actual if m["name"]==desired["name"]) + if found["protocolMapper"]!=desired["protocolMapper"] or any(found["config"].get(k)!=v for k,v in desired["config"].items()): + raise ValueError("foreign relay scope configuration") + # Add this specific pre-approved scope to authenticated DCR. Never allow + # clients to author their own role or audience mappers. + policies=kc("/components?"+urlencode({"type":"org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy"})) + allowed=[p for p in policies if p.get("providerId")=="allowed-client-templates" and p.get("subType")=="authenticated"] + if len(allowed)!=1: + raise ValueError("expected one authenticated client-scope registration policy") + policy=allowed[0] + config=policy.setdefault("config",{}) + permitted=config.setdefault("allowed-client-scopes",[]) + if SCOPE not in permitted: + permitted.append(SCOPE) + kc("/components/"+policy["id"],"PUT",policy) + + def lookup(client_id): + return next((c for c in kc("/clients?"+urlencode({"clientId":client_id})) if c["clientId"]==client_id), None) + settings=dict(clientId=NAME, enabled=True, protocol="openid-connect", publicClient=False, + serviceAccountsEnabled=True,standardFlowEnabled=False,implicitFlowEnabled=False, + directAccessGrantsEnabled=False,fullScopeAllowed=False) + if not lookup(NAME): + kc("/clients","POST",{**settings,"secret":values["client-secret"],"attributes":{"saw.component":NAME}, + "defaultClientScopes":["basic","roles"],"optionalClientScopes":[]}) + client=kc("/clients/"+lookup(NAME)["id"]) + if client.get("attributes",{}).get("saw.component")!=NAME or any(client.get(k)!=v for k,v in settings.items()): + raise ValueError("refusing to adopt a foreign provisioning client") + prefix="/clients/"+client["id"] + management=lookup("realm-management")["id"] + role=kc("/clients/"+management+"/roles/create-client") + user=kc(prefix+"/service-account-user")["id"] + for path in [prefix+"/scope-mappings/clients/"+management, + "/users/"+user+"/role-mappings/clients/"+management]: + current=kc(path) + if {r["name"] for r in current} - {"create-client"}: + raise ValueError("provisioner has unexpected realm-management privileges") + if not current: + kc(path,"POST",[role]) + result=call(issuer+"/protocol/openid-connect/token","POST",form={ + "client_id":NAME,"client_secret":values["client-secret"],"grant_type":"client_credentials"}) + part=result["access_token"].split(".")[1] + claim=json.loads(base64.urlsafe_b64decode(part+"="*(-len(part)%4))) + if set(claim.get("resource_access",{}).get("realm-management",{}).get("roles",[]))!={"create-client"}: + raise ValueError("unexpected effective registrar privilege") + print("Realm registrar verified: create-client in the workspace realm.") + + +def render(args): + ns=args.namespace + metadata={"name":NAME,"namespace":ns} + subject={"kind":"ServiceAccount","name":NAME,"namespace":ns} + objects=[ + {"apiVersion":"v1","kind":"Secret","metadata":{**metadata,"labels":{"saw.redhat.com/component":NAME}},"type":"Opaque"}, + {"apiVersion":"v1","kind":"ServiceAccount","metadata":metadata}, + {"apiVersion":"rbac.authorization.k8s.io/v1","kind":"Role","metadata":metadata,"rules":[ + {"apiGroups":[""],"resources":["secrets"],"verbs":["get","patch"],"resourceNames":[NAME]}]}, + {"apiVersion":"rbac.authorization.k8s.io/v1","kind":"RoleBinding","metadata":metadata,"subjects":[subject], + "roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"Role","name":NAME}}, + {"apiVersion":"v1","kind":"ConfigMap","metadata":metadata,"data":{"setup.py":Path(__file__).read_text()}}] + volumes=[{"name":"script","configMap":{"name":NAME}},{"name":"bootstrap","secret":{ + "secretName":args.bootstrap_secret,"items":[{"key":k,"path":k} for k in ("username","password")]}}] + mounts=[{"name":v["name"],"mountPath":"/"+v["name"],"readOnly":True} for v in volumes] + env={"ISSUER":args.issuer,"NAMESPACE":ns,"AUDIENCE":args.audience} + if args.ca_configmap: + volumes.append({"name":"trust","configMap":{"name":args.ca_configmap}}) + mounts.append({"name":"trust","mountPath":"/trust","readOnly":True}) + env["SSL_CERT_FILE"]="/trust/ca-bundle.crt" + objects.append({"apiVersion":"batch/v1","kind":"Job","metadata":metadata,"spec":{ + "activeDeadlineSeconds":180,"backoffLimit":1,"ttlSecondsAfterFinished":900, + "template":{"spec":{"serviceAccountName":NAME,"restartPolicy":"Never","volumes":volumes, + "securityContext":{"runAsNonRoot":True,"seccompProfile":{"type":"RuntimeDefault"}}, + "containers":[{"name":"setup","image":IMAGE,"command":["python3","/script/setup.py","run"], + "volumeMounts":mounts,"env":[{"name":k,"value":v} for k,v in env.items()], + "securityContext":{"readOnlyRootFilesystem":True,"allowPrivilegeEscalation":False,"capabilities":{"drop":["ALL"]}}}]}}}}) + print(json.dumps({"apiVersion":"v1","kind":"List","items":objects})) + + +if __name__=="__main__": + if sys.argv[1:]==["run"]: + try: + setup() + except HTTPError as exc: + sys.exit(f"Setup API failed: HTTP {exc.code} at {urlsplit(exc.url).path}") + else: + parser=argparse.ArgumentParser(description=__doc__) + for flag in ("namespace","issuer","bootstrap-secret"): + parser.add_argument("--"+flag,required=True) + parser.add_argument("--audience",default="openshell-cli") + parser.add_argument("--ca-configmap") + render(parser.parse_args()) diff --git a/scripts/saw-profile-catalog.py b/scripts/saw-profile-catalog.py index eb708760f..d6d6c7d29 100644 --- a/scripts/saw-profile-catalog.py +++ b/scripts/saw-profile-catalog.py @@ -23,6 +23,9 @@ gets when governance is off; a file no longer shipped is removed) +Canonical workspace projection and Keycloak bootstrap helpers are mirrored +into the independently installable RHDH chart by the same generation/check. + scripts/saw-profile-catalog.py write both files scripts/saw-profile-catalog.py --check fail when they are stale (CI) scripts/saw-profile-catalog.py --list print the files it owns (writes, or removes) @@ -167,6 +170,7 @@ def catalog(): out = {} for pdir in sorted(p for p in PROFILES.iterdir() if p.is_dir()): workspaces, secrets, descriptions = [], {}, [] + endpoint_secrets = set() # The default workspace first: its description is the profile's. for wdir in sorted((w for w in pdir.iterdir() if w.is_dir()), key=lambda w: (w.name != "default", w.name)): @@ -182,9 +186,15 @@ def catalog(): required_string(sb, "name", (wdir / "sandbox.yaml").relative_to(ROOT)) ui = sb.get("ui") or {} sandboxes.append({"name": sb["name"], "type": sb.get("type", "generic"), - "enabled": sb.get("enabled", True), + "enabled": sb.get("enabled", True), "image": sb.get("image", ""), "uiRoute": bool(ui.get("route", False)), "harnessRequired": bool(sb.get("harnessRef") and sb.get("harnessRequired"))}) + model_ref = secret_ref((sb.get("inference") or {}).get("model")) + endpoint_ref = secret_ref((sb.get("inference") or {}).get("baseUrl")) + if endpoint_ref and sb["inference"].get("providerRef") == "custom-inference": + sandboxes[-1]["endpointInput"] = ".".join(endpoint_ref) + if model_ref: + sandboxes[-1]["modelInput"] = ".".join(model_ref) workspaces.append({"name": name, "description": description, "enabled": enabled, "sandboxes": sandboxes}) if not enabled: @@ -219,6 +229,13 @@ def catalog(): fields = list(sandbox_fields(sb, {p["name"]: p for p in providers})) except ValueError as exc: raise ValueError(f"{(wdir / 'sandbox.yaml').relative_to(ROOT)}: {exc}") from None + if secret_ref(inference.get("baseUrl")) and inference.get("providerRef") == "custom-inference": + provider = next((p for p in providers if p["name"] == inference["providerRef"] + and p.get("enabled", True)), None) + if not provider or not provider.get("credentialSecret"): + raise ValueError("dedicated endpoint requires a configured inference credential") + endpoint_secrets.add(provider["credentialSecret"]) + endpoint_secrets.update(secret for secret, _ in fields) for secret, f in fields: entry = secrets.setdefault(secret, {"providers": [], "fields": []}) if f["key"] not in {x["key"] for x in entry["fields"]}: @@ -232,6 +249,8 @@ def catalog(): "harnessRequired": any(sb["enabled"] and sb["harnessRequired"] for ws in workspaces if ws["enabled"] for sb in ws["sandboxes"])} + if endpoint_secrets: + out[pdir.name]["dedicatedEndpointSecrets"] = sorted(endpoint_secrets) return {"profiles": out} @@ -286,13 +305,13 @@ def log_step(step_id, name, source): CREATE_TASKS = [ ("register", "Verify the request, store the keys, register the workspace", 5), ("argo-cd-apps", "Argo CD creates the workspace's applications", 12), - ("vm", "Argo CD creates the VM", 12), - ("vm-running", "Start the VM", 17), - ("sandboxes", "Install OpenShell and the sandboxes (about 10 minutes)", 32), + ("vm", "Provision the workspace runtime", 12), + ("vm-running", "Start the workspace runtime", 17), + ("sandboxes", "Wait for OpenShell sandboxes and workspace access", 32), ] DELETE_TASKS = [ ("unregister", "Verify the request, delete the Argo CD application", 5), - ("argo-cd-removes", "Argo CD removes the namespace and the VM", 22), + ("argo-cd-removes", "Wait for Argo CD to remove the workspace runtime", 22), ("finish", "Remove the registry entry and the keys", 5), ] @@ -346,7 +365,8 @@ def render_template(cat, for_user=False): the workspace, until its sandbox UIs answer), then the pipeline's log.""" profiles = cat["profiles"] one_of, string_data = [], {"action": "create", "token": "${{ secrets.backstageToken }}", - "profile": "${{ parameters.profile }}"} + "profile": "${{ parameters.profile }}", + "deploymentMode": "${{ parameters.deploymentMode }}"} if for_user: string_data["forUser"] = FOR_USER who = FOR_USER if for_user else USER_NAME @@ -377,7 +397,16 @@ def render_template(cat, for_user=False): props[name] = prop if field.get("required"): required.append(name) - one_of.append({"properties": props, "required": required}) + case = {"properties": props, "required": required} + if "dedicatedEndpointSecrets" in prof: + excluded = {form_name(sname, field["key"]) for sname, item in prof["secrets"].items() + if sname not in prof["dedicatedEndpointSecrets"] for field in item["fields"]} + if excluded: + vm_fields = {key: props.pop(key) for key in sorted(excluded)} + case["required"] = [key for key in required if key not in excluded] + case["allOf"] = [{"if": {"properties":{"deploymentMode":{"const":"vm"}}}, + "then": {"properties":vm_fields, "required":[key for key in required if key in excluded]}}] + one_of.append(case) names = sorted(profiles) template = { "apiVersion": "scaffolder.backstage.io/v1beta3", @@ -389,7 +418,7 @@ def render_template(cat, for_user=False): "description": ("Administrators: a Secure Agent Workspace for a user, yourself included, with " "the keys you enter for them; for a user who has one, it changes its profile " "or keys. Keys go to Vault, not to Git." if for_user else - "Your own Secure Agent Workspace: a VM with OpenShell and the OpenClaw / " + "Your own Secure Agent Workspace with OpenShell and the OpenClaw / " "NemoClaw sandboxes of a SAW-BOM profile. If you have one, this changes its " "profile or keys. Keys go to Vault, not to Git."), "tags": ["openshell", "openclaw", "nemoclaw", "secure-agent-workspace"] @@ -406,8 +435,13 @@ def render_template(cat, for_user=False): "title": "Workspace profile", "description": ("The workspace is named after the user (namespace saw-)." if for_user else "The workspace is named after you (namespace saw-)."), - "required": ["profile"], - "properties": {"profile": { + "required": ["profile", "deploymentMode"], + "properties": {"deploymentMode": { + "title": "Deployment", "type": "string", "enum": ["vm"], + "enumNames": ["Virtual machine"], "default": "vm", + "description": "Available deployment options are set by your administrator. " + "An existing workspace keeps its deployment type."}, + "profile": { "title": "Profile", "type": "string", "enum": names, "enumNames": [f"{n}: {profiles[n]['description']}" if profiles[n]["description"] else n for n in names], @@ -441,6 +475,23 @@ def render_template(cat, for_user=False): + yaml.safe_dump(template, sort_keys=False, width=100, default_flow_style=None)) +def render_lifecycle_template(catalog): + """Use the owner create pipeline for authentication, admission, progress and logs.""" + template = yaml.safe_load(render_template(catalog)) + template["metadata"].update(labels={"saw.redhat.com/owners": "true"}, name="set-saw-workspace-state", title="Stop or start my Container or Kata workspace", + description="Pause or resume all sandboxes while retaining workspace files and audit history.") + spec = template["spec"] + spec["parameters"] = [{"title": "Workspace state", "required": ["operation"], + "properties": {"operation": {"type": "string", "enum": ["stop", "start"]}}}] + request = spec["steps"][0]["input"]["body"]["stringData"] + spec["steps"][0]["input"]["body"]["stringData"] = { + "action": "create", "token": request["token"], "runtimeAction": "${{ parameters.operation }}", + "registration": "${{ steps.current.output.body.registration }}", + "expectedGeneration": "g-${{ steps.current.output.body.generation }}"} + spec["steps"].insert(0, status_get("current", "Read my current registration", "workspace")) + return "# Generated by scripts/saw-profile-catalog.py; do not edit.\n" + yaml.safe_dump(template, sort_keys=False, width=100, default_flow_style=None) + + def render_delete_template(for_user=False): """The RHDH scaffolder template that deletes the user's workspace (an administrator's: any user's).""" @@ -451,11 +502,11 @@ def render_delete_template(for_user=False): "metadata": { "name": "delete-saw-workspace-for-user" if for_user else "delete-saw-workspace", "title": "Delete a user's agent workspace" if for_user else "Delete my agent workspace", - "description": ("Administrators: removes a user's Secure Agent Workspace, its VM, its sandboxes " - "and everything in them, and the user's keys in Vault. This cannot be undone." + "description": ("Administrators: removes a user's Secure Agent Workspace, its runtime, its sandboxes " + "and the user's keys in Vault. Retained audit history is separate. This cannot be undone." if for_user else - "Removes your Secure Agent Workspace: its VM, its sandboxes and everything in " - "them, and your keys in Vault. This cannot be undone."), + "Removes your Secure Agent Workspace: its runtime, its sandboxes and the data in " + "them, and your keys in Vault. Retained audit history is separate. This cannot be undone."), "tags": ["openshell", "secure-agent-workspace"] + (["admin"] if for_user else []), # Administrators only; the user's: users with a workspace. "labels": {"saw.redhat.com/admin" if for_user else "saw.redhat.com/owners": "true"}, @@ -478,7 +529,7 @@ def render_delete_template(for_user=False): "spec.type": "agent-workspace"}, "defaultKind": "Component", "allowArbitraryValues": False}}, - "confirm": {"title": "I understand that this workspace, its VM and its data are deleted", + "confirm": {"title": "I understand that the workspace runtime, sandbox data and workspace keys will be deleted", "type": "boolean", "const": True}}, }], "steps": [ @@ -500,7 +551,7 @@ def render_delete_template(for_user=False): ], "output": {"text": [{"title": "Workspace deleted", "content": "Pipeline run **" + RUN_NAME + "** deleted workspace " + picked - + ": its VM, namespace and keys are gone."}]}, + + ": its workspace compute and access have been removed."}]}, }, } return ("# Generated by scripts/saw-profile-catalog.py; do not edit.\n" @@ -518,12 +569,25 @@ def main(argv=None): mirror = {INSTALLER_PROFILES / src.name: src.read_text(encoding="utf-8") for src in sorted(GOVERNANCE_PROFILES.glob("*.yaml"))} extra = [p for p in sorted(INSTALLER_PROFILES.glob("*.yaml")) if p not in mirror] + lifecycle = TEMPLATE_OUTPUT.with_name("set-workspace-state.yaml") + # Independent Helm charts package the same reviewed broker renderer. + policy_template = ROOT / "charts/openshell-saw/templates/_governed-proxy-policy.tpl" + policy_outputs = [ + (ROOT / "charts/openshell-rhdh/files/saw_keycloak_admin.py", + (ROOT / "charts/openshell-keycloak/files/saw_keycloak_admin.py").read_text()), + (ROOT / "charts/saw-workspace-relay/files/saw_ui_limit.py", + (ROOT / "charts/openshell-saw/files/installer/saw_ui_limit.py").read_text()), + (ROOT / "charts/openshell-rhdh/files/workspace.py", + (ROOT / "charts/saw-users/files/workspace.py").read_text()), + (ROOT / "charts/saw-users/templates/_governed-proxy-policy.tpl", policy_template.read_text()), + (ROOT / "charts/saw-users/files/governed-profiles.json", json.dumps( + {p.stem: load(p) for p in sorted(GOVERNANCE_PROFILES.glob("*.yaml"))}, sort_keys=True, indent=2) + "\n")] if args.list: # Every path this script owns: the generated files, each mirror, and # the mirror directory itself, so `git add -A -- ` (the hook, # the workflow) also stages a mirror this run removed. - for path in (*CATALOG_OUTPUTS, TEMPLATE_OUTPUT, DELETE_TEMPLATE, - ADMIN_TEMPLATE_OUTPUT, ADMIN_DELETE_TEMPLATE, *mirror, INSTALLER_PROFILES): + for path in (*CATALOG_OUTPUTS, TEMPLATE_OUTPUT, lifecycle, DELETE_TEMPLATE, + ADMIN_TEMPLATE_OUTPUT, ADMIN_DELETE_TEMPLATE, *(p for p, _ in policy_outputs), *mirror, INSTALLER_PROFILES): print(path.relative_to(ROOT)) return 0 try: @@ -533,9 +597,11 @@ def main(argv=None): return 1 outputs = [(path, text) for path in CATALOG_OUTPUTS] outputs.append((TEMPLATE_OUTPUT, render_template(json.loads(text)))) + outputs.append((lifecycle, render_lifecycle_template(json.loads(text)))) outputs.append((DELETE_TEMPLATE, render_delete_template())) outputs.append((ADMIN_TEMPLATE_OUTPUT, render_template(json.loads(text), for_user=True))) outputs.append((ADMIN_DELETE_TEMPLATE, render_delete_template(for_user=True))) + outputs += policy_outputs outputs += sorted(mirror.items()) stale = [] for path in extra: diff --git a/tests/charts/test_agent_sandbox_chart.py b/tests/charts/test_agent_sandbox_chart.py new file mode 100644 index 000000000..9376bfe81 --- /dev/null +++ b/tests/charts/test_agent_sandbox_chart.py @@ -0,0 +1,130 @@ +"""Pin and render the optional stock Agent Sandbox platform dependency.""" +import hashlib +import shutil +from pathlib import Path + +import pytest +import yaml +from helm_cache import run_helm + +ROOT = Path(__file__).resolve().parents[2] +CHART = ROOT / "charts" / "agent-sandbox" +HELM = shutil.which("helm") +LOCK = yaml.safe_load((CHART / "source-lock.yaml").read_text()) + + +def upstream(): + return [d for d in yaml.safe_load_all( + (CHART / LOCK["upstream"]["manifestPath"]).read_text()) if d] + + +def render(*args, namespace="agent-sandbox-system", enabled=True): + if not HELM: + pytest.skip("helm is not installed") + return run_helm([HELM, "template", "agent-sandbox", str(CHART), + "-n", namespace, "--set", f"enabled={str(enabled).lower()}", *args], + capture_output=True, text=True) + + +@pytest.fixture(scope="module") +def docs(): + result = render() + assert result.returncode == 0, result.stderr + return [d for d in yaml.safe_load_all(result.stdout) if d] + + +def by_kind(docs, kind): + matches = [d for d in docs if d["kind"] == kind] + assert len(matches) == 1 + return matches[0] + + +def test_vendored_artifact_matches_release_lock(): + data = (CHART / LOCK["upstream"]["manifestPath"]).read_bytes() + assert hashlib.sha256(data).hexdigest() == LOCK["upstream"]["manifestSHA256"] + assert LOCK["upstream"]["commit"] == "b1277676f8bf0e82ea299928997ecf5d696a92f1" + assert LOCK["controller"]["digest"] == \ + "sha256:ef1a1a3076afc996e711c44b388fbb7ffb7cd6858953d7e685166b299d77bd38" + assert "Apache License" in (CHART / LOCK["upstream"]["licensePath"]).read_text() + + +def test_disabled_by_default(): + assert yaml.safe_load((CHART / "values.yaml").read_text())["enabled"] is False + result = render(enabled=False) + assert result.returncode == 0, result.stderr + assert list(yaml.safe_load_all(result.stdout)) == [] + + +def test_only_stock_core_resources_are_installed(docs): + assert {d["kind"] for d in docs} == { + "ServiceAccount", "ClusterRole", "ClusterRoleBinding", + "Service", "Deployment", "CustomResourceDefinition"} + assert len(docs) == 6 + for doc in docs: + if "namespace" in doc["metadata"]: + assert doc["metadata"]["namespace"] == "agent-sandbox-system" + assert by_kind(docs, "ClusterRoleBinding")["subjects"] == [{ + "kind": "ServiceAccount", "name": "agent-sandbox-controller", + "namespace": "agent-sandbox-system"}] + + +def test_crd_and_rbac_remain_upstream(docs): + original = upstream() + assert by_kind(docs, "CustomResourceDefinition")["spec"] == \ + by_kind(original, "CustomResourceDefinition")["spec"] + assert by_kind(docs, "ClusterRole")["rules"] == by_kind(original, "ClusterRole")["rules"] + role = by_kind(docs, "ClusterRole") + assert not any("secrets" in rule["resources"] for rule in role["rules"]) + assert by_kind(docs, "ClusterRoleBinding")["roleRef"]["kind"] == "ClusterRole" + + +def test_served_api_matches_reviewed_openshell_fields(docs): + spec = by_kind(docs, "CustomResourceDefinition")["spec"] + assert spec["group"] == "agents.x-k8s.io" + assert len(spec["versions"]) == 1 + version = spec["versions"][0] + assert version["name"] == "v1beta1" and version["served"] and version["storage"] + properties = version["schema"]["openAPIV3Schema"]["properties"]["spec"]["properties"] + assert set(LOCK["consumer"]["reviewedFields"]) <= properties.keys() + assert properties["operatingMode"]["enum"] == ["Running", "Suspended"] + assert "agentSocket" not in properties # Documented unsupported optional consumer field. + assert LOCK["consumer"]["qualification"] == "source-reviewed-only" + + +def test_stock_image_with_process_health_and_restricted_configuration(docs): + pod = by_kind(docs, "Deployment")["spec"]["template"]["spec"] + container = pod["containers"][0] + assert container["image"] == LOCK["controller"]["repository"] + "@" + LOCK["controller"]["digest"] + assert container["args"] == ["--leader-elect=true"] + assert container["livenessProbe"]["httpGet"] == {"path": "/healthz", "port": "healthz"} + assert container["readinessProbe"]["httpGet"] == {"path": "/readyz", "port": "healthz"} + assert pod["securityContext"] == {"runAsNonRoot": True, "seccompProfile": {"type": "RuntimeDefault"}} + assert container["securityContext"] == {"allowPrivilegeEscalation": False, "capabilities": {"drop": ["ALL"]}} + assert "runAsUser" not in pod["securityContext"] + assert container["resources"]["limits"]["memory"] == "512Mi" + + +def test_retention_and_in_application_ordering(docs): + for kind, wave in [("CustomResourceDefinition", "-2"), + ("ClusterRole", "-1"), ("Deployment", "0")]: + annotations = by_kind(docs, kind)["metadata"]["annotations"] + assert annotations["argocd.argoproj.io/sync-wave"] == wave + if kind == "CustomResourceDefinition": + assert annotations["helm.sh/resource-policy"] == "keep" + assert annotations["argocd.argoproj.io/sync-options"] == "Prune=false,Delete=false" + + +def test_mirror_keeps_locked_digest(): + result = render("--set", "controller.imageRepository=mirror.example.test:5000/platform/agent-sandbox") + assert result.returncode == 0, result.stderr + docs = [d for d in yaml.safe_load_all(result.stdout) if d] + image = by_kind(docs, "Deployment")["spec"]["template"]["spec"]["containers"][0]["image"] + assert image == "mirror.example.test:5000/platform/agent-sandbox@" + LOCK["controller"]["digest"] + + +@pytest.mark.parametrize("args,namespace", [ + ([], "workspace-alice"), + (["--set", "controller.imageRepository=mirror.example.test/controller:latest"], "agent-sandbox-system"), +]) +def test_invalid_namespace_or_mutable_image_is_rejected(args, namespace): + assert render(*args, namespace=namespace).returncode != 0 diff --git a/tests/charts/test_aipcc_openclaw_image.py b/tests/charts/test_aipcc_openclaw_image.py index 7a0de8816..7ba80f0ed 100644 --- a/tests/charts/test_aipcc_openclaw_image.py +++ b/tests/charts/test_aipcc_openclaw_image.py @@ -1,4 +1,4 @@ -"""The openclaw sandboxes run the AIPCC OpenClaw image as is. +"""The shared BOM pins AIPCC OpenClaw. Container sites add UID traversal at build time. It replaced openclaw-openshell, an image of our own built on it. What that image added is now covered elsewhere: @@ -11,8 +11,11 @@ /usr/bin/node-* covers. - The @openclaw/openshell-sandbox plugin is not in it; nothing here uses it. """ +import shutil from pathlib import Path +from helm_cache import run_helm + import pytest import yaml @@ -33,7 +36,10 @@ def test_openclaw_sandboxes_use_the_aipcc_image(): found = list(openclaw_sandboxes()) assert found for path, sb in found: - assert sb["image"].startswith(AIPCC), f"{path}: {sb['image']}" + if path.relative_to(ROOT / "charts/saw-bom/profiles").parts[0] == "audited-openclaw": + assert sb["image"] == "ghcr.io/openclaw/openclaw@sha256:0a5ff5e682e62afa19149df126aa50063bf65ef885b5c94713ce32dc0eb12e15" + else: + assert sb["image"].startswith(AIPCC), f"{path}: {sb['image']}" @pytest.mark.ci_external @@ -49,3 +55,24 @@ def test_the_policy_lets_the_sandbox_read_openclaw_under_opt(): def test_no_image_of_our_own_is_built(): assert not (ROOT / "image-builder-charts" / "helm" / "openclaw-openshell-image").exists() assert "openclaw-openshell" not in (ROOT / "Makefile-quickstart").read_text() + + +@pytest.mark.parametrize("trigger", [True, False]) +def test_container_notebook_builder_preserves_image_user_and_contents(trigger): + helm = shutil.which("helm") + if not helm: + pytest.skip("helm is not installed") + image = next(sb["image"] for path, sb in openclaw_sandboxes() + if path.parent.name == "default" and path.parent.parent.name == "data-science" + and sb["name"] == "notebook") + options = [] if trigger else ["--set", "configChangeTrigger=false", + "--set-string", "buildArchitecture=amd64"] + out = run_helm([helm, "template", "test", str(ROOT / "image-builder-charts/helm/openshift-home-access"), + "--set-string", "images[0].name=notebook", "--set-string", "images[0].from="+image, + "--set-string", "images[0].imageUser=sandbox:sandbox", *options], + capture_output=True, text=True, check=True) + build = next(d["spec"] for d in yaml.safe_load_all(out.stdout) if d and d["kind"] == "BuildConfig") + assert build["source"]["dockerfile"].splitlines() == [ + "FROM "+image, "USER 0", "RUN test -d /sandbox && chmod o+x /sandbox", "USER sandbox:sandbox"] + assert build["triggers"] == ([{"type": "ConfigChange"}] if trigger else []) + assert build.get("nodeSelector") == (None if trigger else {"kubernetes.io/arch": "amd64"}) diff --git a/tests/charts/test_enrollment_api_headers.py b/tests/charts/test_enrollment_api_headers.py new file mode 100644 index 000000000..559d307cb --- /dev/null +++ b/tests/charts/test_enrollment_api_headers.py @@ -0,0 +1,29 @@ +"""Enrollment uses JSON headers that match each Kubernetes API method.""" +import importlib.util +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "enrollment_kube_api", ROOT / "charts/saw-portal-container/files/kube_api.py") +KUBE_API = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(KUBE_API) + + +def test_get_uses_accept_without_a_request_content_type(): + headers = KUBE_API.request_headers("test-token") + + assert headers == { + "Authorization": "Bearer test-token", + "Accept": "application/json", + } + + +def test_patch_uses_merge_patch_content_type(): + headers = KUBE_API.request_headers("test-token", patch=True) + + assert headers == { + "Authorization": "Bearer test-token", + "Accept": "application/json", + "Content-Type": "application/merge-patch+json", + } diff --git a/tests/charts/test_enrollment_http_errors.py b/tests/charts/test_enrollment_http_errors.py new file mode 100644 index 000000000..0495ad69a --- /dev/null +++ b/tests/charts/test_enrollment_http_errors.py @@ -0,0 +1,44 @@ +"""Enrollment diagnostics expose API status and path only.""" +import importlib.util +import io +from pathlib import Path +from urllib.error import HTTPError + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "enrollment_http_errors", ROOT / "charts/saw-portal-container/files/http_errors.py") +HTTP_ERRORS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(HTTP_ERRORS) + + +def test_http_error_shows_status_and_kubernetes_api_path_only(): + error = HTTPError( + "https://kubernetes.default.svc/api/v1/namespaces/saw-ctl-fixture/configmaps/gateway-ca?token=query-secret", + 403, + "private response body", + {"Authorization": "header-secret"}, + io.BytesIO(b"body-secret"), + ) + + result = HTTP_ERRORS.format_http_error(error) + + assert result == "HTTP 403 /api/v1/namespaces/saw-ctl-fixture/configmaps/gateway-ca" + assert all(secret not in result for secret in ( + "kubernetes.default.svc", "query-secret", "private response body", + "header-secret", "body-secret")) + + +def test_http_error_hides_non_kubernetes_urls(): + error = HTTPError("https://user:password@external.example.test/private?token=secret", + 401, "private", {}, io.BytesIO(b"secret")) + + assert HTTP_ERRORS.format_http_error(error) == "HTTP 401" + + +def test_http_error_without_a_valid_status_is_type_only(): + class InvalidError: + code = 700 + url = "https://kubernetes.default.svc/api/v1/secrets/private" + + assert HTTP_ERRORS.format_http_error(InvalidError()) == "HTTPError" diff --git a/tests/charts/test_governance_interceptor_image.py b/tests/charts/test_governance_interceptor_image.py index 089cf61fa..b270c13d7 100644 --- a/tests/charts/test_governance_interceptor_image.py +++ b/tests/charts/test_governance_interceptor_image.py @@ -15,16 +15,11 @@ DOCKERFILE = CHART / "files/Dockerfile" HELM = shutil.which("helm") -# Source patches must remain in both build paths. +EXTENSION = CHART / "files/saw-extension" PATCH_LINES = [ - # Existing workaround for NVIDIA/OpenShell#3929. - "! grep -q 'insert(PROFILE_HASH_ANNOTATION' src/main.rs", - # Agent driver-config guard: admin-only exact mount combinations. - "&& grep -q 'let gate = validate_sandbox_driver_config(' src/main.rs", - "'fn validate_driver_config('", - "'const SAW_ADMIN_ROLE: &str = \"openshell-admin\";'", - "' return deny(\"driver config may only be submitted by a platform admin mTLS identity\");'", - "' return deny(\"driver config must contain gate and audit volumes, with an optional read-only harness\");'", + 'COPY saw-extension/driver_config.rs src/saw_driver_config.rs', + 'COPY saw-extension/integration.patch /tmp/saw-integration.patch', + 'git -C /build apply --check --directory="${CONTEXT_DIR}"', ] @@ -35,6 +30,16 @@ def inline_dockerfile(): return bc["spec"]["source"]["dockerfile"] +@pytest.mark.skipif(not HELM, reason="helm is not installed") +@pytest.mark.parametrize("enabled", [False, True]) +def test_gitops_build_trigger_is_explicit_opt_in(enabled): + out = run_helm([HELM, "template", "img", str(CHART), "--set", + f"build.configChangeTrigger={str(enabled).lower()}"], + capture_output=True, text=True, check=True).stdout + bc = next(d for d in yaml.safe_load_all(out) if d and d["kind"] == "BuildConfig") + assert bc["spec"]["triggers"] == ([{"type": "ConfigChange"}] if enabled else []) + + @pytest.mark.ci_external @pytest.mark.parametrize("line", PATCH_LINES) def test_dockerfile_carries_every_source_patch(line): @@ -52,21 +57,27 @@ def test_buildconfig_carries_every_source_patch(line): def test_the_guard_runs_before_the_build(): """A patch appended after `cargo build` would never reach the binary.""" text = DOCKERFILE.read_text() - assert text.index("validate_driver_config") < text.index("RUN cargo test") + assert text.index("COPY saw-extension/driver_config.rs") < text.index("RUN cargo test") @pytest.mark.ci_external def test_the_guard_admin_role_is_the_role_the_saw_gateways_grant(): - """The guard hardcodes the role name; the gateway takes it from values - (oidc.adminRole, and the OU of the installer's mTLS certificate). They - must not drift, or the installer loses the right to mount a harness.""" + """The guard's OIDC admin role must match the VM installer configuration.""" values = yaml.safe_load((ROOT / "charts" / "openshell-saw" / "values.yaml").read_text()) role = values["oidc"]["adminRole"] - assert f"'const SAW_ADMIN_ROLE: &str = \"{role}\";'" in DOCKERFILE.read_text() + assert f'const SAW_ADMIN_ROLE: &str = "{role}";' in (EXTENSION / "driver_config.rs").read_text() installer = (ROOT / "charts" / "openshell-saw" / "files" / "installer" / "apply_bom.py").read_text() assert f'ADMIN_CERT_SUBJECT = "/O=openshell/OU={role}/CN=saw-installer"' in installer +def test_kubernetes_gateway_uses_mtls_operator_authority(): + config = (ROOT / "charts" / "openshell-saw-kubernetes" / "templates" / "gateway-config.yaml").read_text() + sessions = (ROOT / "kubernetes-runtime" / "saw_kubernetes" / "admin_session.py").read_text() + assert "operator_enabled = true" in config + assert "enabled = false" in config + assert 'OPERATOR_SUBJECT = "/O=openshell/OU=operator/CN=saw-installer"' in sessions + + @pytest.mark.ci_external def test_openshift_builder_check_runs_in_every_platform_build(): workflow = yaml.safe_load((ROOT / ".github/workflows/build-governance-interceptor.yml").read_text()) @@ -97,8 +108,21 @@ def test_gateway_allows_driver_guard_without_noninterceptable_template_rpc(): def appended_rust(text): - block = text.split("RUN printf '%s\\n'", 1)[1].split(">> src/main.rs", 1)[0] - return "\n".join(re.findall(r"^\s*'(.*)'(?: \\)?$", block, re.MULTILINE)) + "\n" + # Dockerfile references the module; OpenShift mounts the same files as context. + assert "COPY saw-extension/driver_config.rs src/saw_driver_config.rs" in text + return "mod saw_driver_config {\n" + (EXTENSION / "driver_config.rs").read_text() + "\n}\n" + + +def test_openshift_build_context_contains_exact_extension_sources(): + rendered = run_helm([HELM, "template", "img", str(CHART)], + capture_output=True, text=True, check=True).stdout + docs = [d for d in yaml.safe_load_all(rendered) if d] + config, = [d for d in docs if d["kind"] == "ConfigMap"] + build, = [d for d in docs if d["kind"] == "BuildConfig"] + assert config["data"] == {p.name: p.read_text() for p in EXTENSION.iterdir()} + assert build["spec"]["source"]["configMaps"] == [ + {"configMap": {"name": config["metadata"]["name"]}, "destinationDir": "saw-extension"}] + assert "saw_driver_config::validate_sandbox_driver_config" in config["data"]["integration.patch"] @pytest.mark.parametrize("builder", ["dockerfile", "buildconfig"]) @@ -196,3 +220,19 @@ def test_default_build_output_is_internal_and_push_requires_explicit_destination "--set", "output.imageStreamTag=", "--set", "output.externalImage=ghcr.io/example/governance-interceptor:v0.1.6"], text=True) build = next(d for d in yaml.safe_load_all(rendered) if d and d["kind"] == "BuildConfig") assert build["spec"]["output"]["to"]["name"] == "ghcr.io/example/governance-interceptor:v0.1.6" + + +@pytest.mark.parametrize("filename", ["build-governance-interceptor.yml", "build-kubernetes-runtime.yml"]) +def test_pr_image_jobs_cannot_publish(filename): + workflow = yaml.safe_load((ROOT / ".github/workflows" / filename).read_text()) + assert workflow["permissions"] == {"contents": "read"} + jobs = workflow["jobs"] + assert "permissions" not in jobs["build"] + assert jobs["build"]["if"] == "github.event_name == 'pull_request' || github.ref != 'refs/heads/main'" + for job in ("publish", "merge"): + assert jobs[job]["if"] == "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" + assert jobs[job]["permissions"] == {"contents": "read", "packages": "write"} + assert jobs["merge"]["needs"] == "publish" + assert jobs["build"]["steps"] == jobs["publish"]["steps"] + image = next(s for s in jobs["build"]["steps"] if s.get("uses") == "./.github/actions/build-push-image") + assert "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" in image["with"]["registry_token"] diff --git a/tests/charts/test_governance_kubernetes_ingress.py b/tests/charts/test_governance_kubernetes_ingress.py new file mode 100644 index 000000000..db75ec439 --- /dev/null +++ b/tests/charts/test_governance_kubernetes_ingress.py @@ -0,0 +1,38 @@ +"""Opt-in container gateway ingress cannot widen the existing VM selectors.""" +import shutil +from pathlib import Path +import pytest +import yaml +from helm_cache import run_helm + +ROOT = Path(__file__).resolve().parents[2] +HELM = shutil.which('helm') +pytestmark = pytest.mark.skipif(not HELM, reason='helm required') + + +def render(*args): + return run_helm([HELM, 'template', 'governance-interceptor', str(ROOT / 'charts/governance-interceptor'), *args], capture_output=True, text=True) + + +def peers(output): + obj = next(d for d in yaml.safe_load_all(output) if d and d['kind'] == 'NetworkPolicy') + return obj['spec']['ingress'][0]['from'] + + +def test_default_stays_vm_only_and_opt_in_adds_exact_combined_selector(): + baseline = render() + assert baseline.returncode == 0, baseline.stderr + old = peers(baseline.stdout) + assert len(old) == 2 + result = render('--set', 'kubernetesGateways[0].namespace=saw-k8s-test', '--set', 'kubernetesGateways[0].gatewayId=saw-k8s-test') + assert result.returncode == 0, result.stderr + new = peers(result.stdout) + assert new[:2] == old + assert new[2:] == [{'namespaceSelector': {'matchLabels': {'kubernetes.io/metadata.name': 'saw-k8s-test'}}, 'podSelector': {'matchLabels': {'app.kubernetes.io/instance': 'saw-k8s-test', 'openshell.pattern/role': 'gateway'}}}] + + +@pytest.mark.parametrize('namespace,gateway', [('', 'test'), ('test', ''), ('*', 'test'), ('test', '*'), ('a' * 64, 'test'), ('test', 'a' * 20)]) +def test_rejects_missing_or_broad_identity(namespace, gateway): + result = render('--set-string', f'kubernetesGateways[0].namespace={namespace}', '--set-string', f'kubernetesGateways[0].gatewayId={gateway}') + assert result.returncode != 0 + assert 'exact namespace and gatewayId' in result.stderr diff --git a/tests/charts/test_openshell_keycloak_chart.py b/tests/charts/test_openshell_keycloak_chart.py index 78f7207ff..64aa64bca 100644 --- a/tests/charts/test_openshell_keycloak_chart.py +++ b/tests/charts/test_openshell_keycloak_chart.py @@ -321,3 +321,30 @@ def test_registrar_http_override_is_rejected(): result = helm("--set", "redirectRegistrar.keycloakUrl=http://sso.example.com") assert result.returncode != 0 assert "must use HTTPS" in result.stderr + + +@pytest.mark.ci_external +def test_shared_admin_scope_helper_is_mounted_and_rolls_bootstraps(tmp_path): + docs = render("--set", "ownerResolver.enabled=true") + names = ("saw-owner-resolver", "saw-redirect-registrar") + source = (CHART / "files/saw_keycloak_admin.py").read_text() + checksums = {} + for name in names: + cm = next(d for d in docs if d["kind"] == "ConfigMap" and d["metadata"]["name"] == name) + assert cm["data"]["saw_keycloak_admin.py"] == source + dep = next(d for d in docs if d["kind"] == "Deployment" and d["metadata"]["name"] == name) + checksums[name] = dep["spec"]["template"]["metadata"]["annotations"]["checksum/script"] + pod = dep["spec"]["template"]["spec"] + assert any("bootstrap" in c.get("command", []) + c.get("args", []) for c in pod["initContainers"]) + assert any(v.get("configMap", {}).get("name") == name for v in pod["volumes"]) + # A helper-only change must rerun existing bootstrap init containers. + chart_copy = tmp_path / "chart" + shutil.copytree(CHART, chart_copy) + (chart_copy / "files/saw_keycloak_admin.py").write_text(source + "\n# changed helper\n") + result = run_helm([HELM, "template", "openshell-keycloak", str(chart_copy), "-n", "keycloak", + "--set", "ownerResolver.enabled=true"], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + changed = [d for d in yaml.safe_load_all(result.stdout) if d] + for name in names: + dep = next(d for d in changed if d["kind"] == "Deployment" and d["metadata"]["name"] == name) + assert dep["spec"]["template"]["metadata"]["annotations"]["checksum/script"] != checksums[name] diff --git a/tests/charts/test_openshell_rhdh_chart.py b/tests/charts/test_openshell_rhdh_chart.py index 5c9c9c568..2557ef936 100644 --- a/tests/charts/test_openshell_rhdh_chart.py +++ b/tests/charts/test_openshell_rhdh_chart.py @@ -28,6 +28,30 @@ def one(docs, kind, name): return found[0] +@pytest.mark.parametrize("minutes,wait,task", [(30, "1800", "32m0s"), (60, "3600", "1h2m0s"), + (58, "3480", "1h0m0s"), (1, "60", "3m0s")]) +def test_bounded_site_guest_readiness_timeout(minutes, wait, task): + docs = render("--set", f"portal.sandboxReadyTimeoutMinutes={minutes}") + pipeline = one(docs, "Pipeline", "saw-workspace-create") + stage = next(t for t in pipeline["spec"]["tasks"] if t["name"] == "sandboxes") + assert stage["timeout"] == task + assert {p["name"]: p["value"] for p in stage["params"]}["timeout"] == wait + assert {p["name"]: p["value"] for p in stage["params"]}["action"] == "wait-ready" + cm = one(docs, "ConfigMap", "saw-rhdh-templates") + for key in ("create-workspace.yaml", "create-workspace-for-user.yaml"): + if key in cm["data"]: + form = yaml.safe_load(cm["data"][key]) + step = next(s for s in form["spec"]["steps"] if s["id"] == "task-sandboxes") + assert step["each"] == list(range(1, (minutes + 2) * 3 + 1)) + + +@pytest.mark.parametrize("minutes", ["0", "61", "false", "1.5", "unlimited"]) +def test_invalid_guest_readiness_timeout_fails_render(minutes): + result = run_helm([HELM, "template", "openshell-rhdh", str(CHART), "-n", "rhdh", + "--set", f"portal.sandboxReadyTimeoutMinutes={minutes}"], capture_output=True, text=True) + assert result.returncode != 0 and not result.stdout.strip() + + @pytest.fixture(scope="module") def docs(): return render() @@ -49,12 +73,20 @@ def test_templates_are_the_generated_form_with_the_provisioner(docs): @pytest.mark.ci_external -def test_every_form_field_reaches_the_request(docs): +@pytest.mark.parametrize("container", [False, True]) +def test_every_form_field_reaches_the_request(docs, container): """The form asks for what the catalog says each profile needs, and the request Secret carries each field under ..""" catalog = json.loads((CHART / "files" / "profile-catalog.json").read_text())["profiles"] create = yaml.safe_load(one(docs, "ConfigMap", "saw-rhdh-templates")["data"]["create-workspace.yaml"]) string_data = create["spec"]["steps"][0]["input"]["body"]["stringData"] + if container: + site = ROOT/"tests/fixtures/container-site.yaml" + with_containers = render("-f",str(site)) + create=yaml.safe_load(one(with_containers,"ConfigMap","saw-rhdh-templates")["data"]["create-workspace.yaml"]) + string_data=create["spec"]["steps"][0]["input"]["body"]["stringData"] + else: + catalog={k:v for k,v in catalog.items() if k!="audited-openclaw"} one_of = create["spec"]["parameters"][0]["dependencies"]["profile"]["oneOf"] assert sorted(o["properties"]["profile"]["const"] for o in one_of) == sorted(catalog) for profile, spec in catalog.items(): @@ -117,6 +149,7 @@ def test_the_applicationset_renders_saw_users_per_workspace(docs): assert aset["spec"]["syncPolicy"] == {"applicationsSync": "create-update", "preserveResourcesOnDeletion": True} assert template["spec"]["source"]["path"] == "charts/saw-users" + assert template["spec"]["source"]["targetRevision"] == "{{ .revision }}" assert template["spec"]["source"]["helm"]["values"] == "{{ .values }}" plugin = one(docs, "ConfigMap", "saw-portal-generator") token = one(docs, "Secret", "saw-portal-generator") @@ -167,10 +200,11 @@ def test_the_pipeline_task_runs_portal_py(docs): assert "VERIFY_TOKEN" not in env values = yaml.safe_load((CHART / "values.yaml").read_text()) assert "verifyToken" not in values["portal"] - assert env["RHDH_INTERNAL_URL"] == "http://backstage-developer-hub.rhdh.svc:80" + assert env["RHDH_INTERNAL_URL"] == "https://backstage-developer-hub-rhdh.apps.example.com" assert env["ARGO_NAMESPACE"] == "vp-gitops" assert env["RESULT_PATH"] == "$(results.user.path)" - assert env["PIPELINE_RUN"] == "$(context.pipelineRun.name)" + run = next(e for e in step["env"] if e["name"] == "PIPELINE_RUN") + assert run["valueFrom"] == {"fieldRef": {"fieldPath": "metadata.labels['tekton.dev/pipelineRun']"}} scripts = one(docs, "ConfigMap", "saw-portal-scripts") assert scripts["data"]["portal.py"] == (CHART / "files" / "portal.py").read_text() @@ -387,7 +421,7 @@ def test_the_generator_may_only_read_progress(docs): ("saw-portal", "", "pods/log", "get"), ("vp-gitops", "argoproj.io", "applications", "get")} env = {e["name"]: e["value"] for e in one(docs, "Deployment", "saw-workspaces-generator") ["spec"]["template"]["spec"]["containers"][0]["env"]} - assert env["ARGO_NAMESPACE"] == "vp-gitops" and env["RHDH_INTERNAL_URL"].startswith("http") + assert env["ARGO_NAMESPACE"] == "vp-gitops" and env["RHDH_INTERNAL_URL"] == env["RHDH_BASE_URL"] @pytest.mark.ci_external @@ -457,7 +491,7 @@ def test_admins_get_templates_for_another_user(): targets = [loc["target"] for loc in config["catalog"]["locations"]] assert "/opt/app-root/src/saw/create-workspace-for-user.yaml" in targets task = one(docs, "Task", "saw-workspace") - env = {e["name"]: e["value"] for e in next(s for s in task["spec"]["steps"] if s["name"] == "run")["env"]} + env = {e["name"]: e.get("value") for e in next(s for s in task["spec"]["steps"] if s["name"] == "run")["env"]} assert env["PORTAL_ADMINS"] == "admin" none = render("--set", "portal.admins=null") assert "create-workspace-for-user.yaml" not in one(none, "ConfigMap", "saw-rhdh-templates")["data"] @@ -520,7 +554,13 @@ def test_tls_is_verified_everywhere_by_default(docs): assert "NODE_TLS_REJECT_UNAUTHORIZED" not in s["node"] assert s["node"]["NODE_EXTRA_CA_CERTS"] == "/opt/saw-ca/ca-bundle.crt" pod = s["patch"]["spec"]["template"]["spec"] - assert [c["name"] for c in pod["initContainers"]] == ["saw-ca-bundle"] + assert [c["name"] for c in pod["initContainers"]] == ["saw-ca-bundle", "saw-oidc-discovery"] + gate = pod["initContainers"][1] + assert gate["command"][2] == "https://openshell-keycloak-ingress-saw-keycloak.apps.example.com/realms/openshell" + assert gate["command"][3] == "/opt/saw-ca/ca-bundle.crt" + assert {v["name"] for v in gate["volumeMounts"]} == {"saw-ca", "saw-ca-sources"} + assert all(v["readOnly"] for v in gate["volumeMounts"]) + assert gate["securityContext"]["allowPrivilegeEscalation"] is False assert pod["containers"][0]["name"] == "backstage-backend" assert {"name": "saw-ca", "mountPath": "/opt/saw-ca", "readOnly": True} in pod["containers"][0]["volumeMounts"] assert s["proxy"] == [True, True] @@ -539,7 +579,9 @@ def test_tls_is_verified_everywhere_by_default(docs): @pytest.mark.ci_external def test_one_switch_turns_verification_off(docs): s = tls_settings(render("--set", "tls.insecureSkipVerify=true")) - assert s["node"]["NODE_TLS_REJECT_UNAUTHORIZED"] == "0" and s["patch"] is None + assert s["node"]["NODE_TLS_REJECT_UNAUTHORIZED"] == "0" + tpl = s["patch"]["spec"]["template"] + assert "spec" not in tpl and len(tpl["metadata"]["annotations"]["saw.redhat.com/portal-templates"]) == 64 assert s["proxy"] == [False, False] assert s["kubernetes"]["skipTLSVerify"] is True and "caFile" not in s["kubernetes"] assert _envs(s["generator"]["containers"][0])["KEYCLOAK_SKIP_VERIFY"] == "true" @@ -635,3 +677,239 @@ def test_portal_workspaces_inherit_shared_credential_mode(tmp_path, enabled): "credentialProxy": {"enabled": enabled}}}})) result = generator_defaults(render("-f", str(path))) assert result["defaults"]["openshellSaw"]["credentialProxy"]["enabled"] == enabled + + +@pytest.mark.ci_external +@pytest.mark.parametrize("modes", [["vm"], ["vm", "container"], ["vm", "kata"]]) +def test_operator_mode_policy_reaches_both_forms_and_server(modes): + docs = render("--set-json", f"portal.deploymentModes={json.dumps(modes)}", + "--set-json", 'portal.admins=["admin"]') + templates = one(docs, "ConfigMap", "saw-rhdh-templates")["data"] + for name in ("create-workspace.yaml", "create-workspace-for-user.yaml"): + form = yaml.safe_load(templates[name]) + field = form["spec"]["parameters"][-1]["properties"]["deploymentMode"] + # An operator allowlist alone does not make a container profile usable. + assert field["enum"] == ["vm"] and field["default"] == "vm" + request = form["spec"]["steps"][0]["input"]["body"]["stringData"] + assert request["deploymentMode"] == "${{ parameters.deploymentMode }}" + assert "runtimeClassName" not in str(form) + task = one(docs, "Task", "saw-workspace") + step = next(s for s in task["spec"]["steps"] if s["name"] == "run") + dep = one(docs, "Deployment", "saw-workspaces-generator") + for container in [step, dep["spec"]["template"]["spec"]["containers"][0]]: + env = {e["name"]: e.get("value") for e in container["env"]} + assert json.loads(env["SAW_DEPLOYMENT_MODES"]) == modes + + +@pytest.mark.ci_external +@pytest.mark.parametrize("modes", [[], ["container"], ["vm", "vm"], ["vm", "confidential"], ["vm", "kata", "kata"], + ["vm", "bogus"], ["vm", {}], "vm"]) +def test_invalid_operator_mode_policy_rejects_chart(modes): + result = run_helm([HELM, "template", "openshell-rhdh", str(CHART), "-n", "rhdh", + "--set-json", f"portal.deploymentModes={json.dumps(modes)}"], + capture_output=True, text=True) + assert result.returncode != 0 + assert "portal.deploymentModes" in result.stderr + assert not result.stdout.strip() + + +def _deployment_profiles(form): + step = form["spec"]["parameters"][-1] + assert "profile" not in step["properties"], "a broad base enum would undo UI filtering" + return step, {branch["properties"]["deploymentMode"]["enum"][0]: + branch["properties"]["profile"] + for branch in step["dependencies"]["deploymentMode"]["oneOf"]} + + +@pytest.mark.ci_external +def test_container_choices_are_approved_catalog_profiles_in_both_forms(): + sites = { + "alice": {"subject": "11111111-1111-1111-1111-111111111111", + "automaticEnrollment": {"approvedProfiles": {"custom-inference": "a" * 64, + "not-in-catalog": "b" * 64}}}, + "bob": {"subject": "22222222-2222-2222-2222-222222222222", + "automaticEnrollment": {"approvedProfiles": {"data-science": "c" * 64}}}, + "unregistered": {"automaticEnrollment": {"approvedProfiles": {"ci-nvidia": "d" * 64}}}, + } + docs = render("--set-json", 'portal.deploymentModes=["vm","container"]', + "--set-json", 'portal.admins=["admin"]', + "--set-json", "portal.containerSites=" + json.dumps(sites)) + templates = one(docs, "ConfigMap", "saw-rhdh-templates")["data"] + generated = yaml.safe_load((CHART / "files/create-workspace.yaml").read_text()) + original = generated["spec"]["parameters"][-1] + for name in ("create-workspace.yaml", "create-workspace-for-user.yaml"): + step, fields = _deployment_profiles(yaml.safe_load(templates[name])) + assert fields["container"]["enum"] == ["custom-inference", "data-science"] + assert fields["container"]["default"] == "custom-inference" + assert "your account" in fields["container"]["description"] + assert fields["vm"]["enum"] == [p for p in original["properties"]["profile"]["enum"] + if p != "audited-openclaw"] + assert fields["vm"]["default"] == original["properties"]["profile"]["default"] + # Credential widgets still come from the single generated BOM form. + assert step["dependencies"]["profile"] == original["dependencies"]["profile"] + for field in fields.values(): + assert len(field["enum"]) == len(field["enumNames"]) + assert all(label.startswith(profile + ":") + for profile, label in zip(field["enum"], field["enumNames"])) + + +@pytest.mark.ci_external +def test_secret_backed_site_approvals_feed_runtime_and_kata_form(): + docs = render("--set-string", "portal.containerSitesSecretName=site-approvals", + "--set-json", 'portal.admins=["admin"]') + secret_ref = {"secretKeyRef": {"name": "site-approvals", "key": "container-sites.json"}} + + task = one(docs, "Task", "saw-workspace") + task_step = next(s for s in task["spec"]["steps"] if s["name"] == "run") + generator = one(docs, "Deployment", "saw-workspaces-generator") + generator_container = generator["spec"]["template"]["spec"]["containers"][0] + for container in (task_step, generator_container): + env = {entry["name"]: entry for entry in container["env"]} + assert env["SAW_CONTAINER_SITES"]["valueFrom"] == secret_ref + assert "value" not in env["SAW_CONTAINER_SITES"] + assert any(volume["name"] == "owner-directory" for volume in task["spec"]["volumes"]) + + templates = one(docs, "ConfigMap", "saw-rhdh-templates")["data"] + for name in ("create-workspace.yaml", "create-workspace-for-user.yaml"): + form = yaml.safe_load(templates[name]) + step, profiles = _deployment_profiles(form) + assert step["properties"]["deploymentMode"]["enum"] == ["vm", "kata"] + assert "data-science" in profiles["kata"]["enum"] + assert profiles["kata"]["default"] in profiles["kata"]["enum"] + + +@pytest.mark.ci_external +def test_kata_preflight_read_access_includes_the_rhdh_generator(): + docs = render("--set-json", 'portal.deploymentModes=["vm","kata"]') + role = one(docs, "ClusterRole", "saw-portal-kata-preflight") + runtime_rule = next(rule for rule in role["rules"] + if rule.get("apiGroups") == ["node.k8s.io"] + and rule.get("resources") == ["runtimeclasses"]) + assert runtime_rule["verbs"] == ["get"] + admission_rule = next(rule for rule in role["rules"] + if rule.get("apiGroups") == ["admissionregistration.k8s.io"] + and set(rule.get("resources", [])) == { + "validatingadmissionpolicies", + "validatingadmissionpolicybindings", + }) + assert admission_rule["verbs"] == ["get"] + + binding = one(docs, "ClusterRoleBinding", "saw-portal-kata-preflight") + subjects = {(subject["kind"], subject["name"], subject["namespace"]) + for subject in binding["subjects"]} + assert subjects == { + ("ServiceAccount", "saw-portal-provisioner", "saw-portal"), + ("ServiceAccount", "saw-workspaces-generator", "saw-portal"), + } + + +@pytest.mark.ci_external +def test_site_approval_values_cannot_mix_inline_data_and_secret_reference(): + result = run_helm([HELM, "template", "openshell-rhdh", str(CHART), "-n", "rhdh", + "--set-json", 'portal.containerSites={"alice":{"subject":"11111111-1111-1111-1111-111111111111"}}', + "--set-string", "portal.containerSitesSecretName=site-approvals"], + capture_output=True, text=True) + assert result.returncode != 0 + assert "set only one of portal.containerSites and portal.containerSitesSecretName" in result.stderr + + +@pytest.mark.ci_external +@pytest.mark.parametrize("automatic,expected", [ + ({"profileSHA256": "a" * 64}, []), + ({"profileSHA256": "a" * 64, "approvedProfiles": {}}, []), + ({"approvedProfiles": {"custom-inference": ""}}, []), + ({"approvedProfiles": {"not-in-catalog": "a" * 64}}, []), +]) +def test_container_choice_requires_known_reviewed_profile(automatic, expected): + sites = {"alice": {"subject": "11111111-1111-1111-1111-111111111111", + "automaticEnrollment": automatic}} + docs = render("--set-json", 'portal.deploymentModes=["vm","container"]', + "--set-json", "portal.containerSites=" + json.dumps(sites)) + form = yaml.safe_load(one(docs, "ConfigMap", "saw-rhdh-templates")["data"]["create-workspace.yaml"]) + step, fields = _deployment_profiles(form) + if expected: + assert fields["container"]["enum"] == expected + assert "container" in step["properties"]["deploymentMode"]["enum"] + else: + assert "container" not in fields + assert "container" not in step["properties"]["deploymentMode"]["enum"] + + +@pytest.mark.ci_external +def test_rhdh_identity_bootstrap_can_repair_unhealthy_consumers(docs): + cm = one(docs, "ConfigMap", "saw-rhdh-keycloak-client") + assert cm["data"]["saw_keycloak_admin.py"] == (CHART / "files/saw_keycloak_admin.py").read_text() + job = one(docs, "Job", "saw-rhdh-keycloak-client") + # A 401 makes the AppSet unhealthy; PostSync would prevent the bootstrap + # that repairs it from running. PreSync would miss first-install inputs. + assert job["metadata"]["annotations"]["argocd.argoproj.io/hook"] == "Sync" + def wave(resource): + return int(resource["metadata"].get("annotations", {}).get("argocd.argoproj.io/sync-wave", "0")) + + secret = one(docs, "ExternalSecret", "rhdh-oidc") + dependencies = [cm, one(docs, "ServiceAccount", "saw-rhdh-keycloak"), secret, + one(docs, "Role", "saw-rhdh-keycloak-admin"), + one(docs, "RoleBinding", "saw-rhdh-keycloak-admin"), + one(docs, "Role", "saw-portal-ingress-ca"), + one(docs, "RoleBinding", "saw-portal-ingress-ca")] + dependencies.extend(r for r in docs if r["kind"] == "ConfigMap" + and r["metadata"]["name"] in ("saw-ca-sources", "saw-trusted-ca") + and r["metadata"]["namespace"] == job["metadata"]["namespace"]) + for resource in dependencies: + assert not resource["metadata"].get("annotations", {}).get("argocd.argoproj.io/hook") + assert wave(resource) < wave(job) + consumers = [r for r in docs if r["kind"] in ("Backstage", "ApplicationSet")] + assert {r["kind"] for r in consumers} == {"Backstage", "ApplicationSet"} + assert all(wave(job) < wave(r) for r in consumers) + client = job["spec"]["template"]["spec"]["containers"][0] + ref = next(e["valueFrom"]["secretKeyRef"] for e in client["env"] if e["name"] == "CLIENT_SECRET") + assert ref == {"name": secret["spec"]["target"]["name"], "key": "AUTH_OIDC_CLIENT_SECRET"} + assert 0 < job["spec"]["activeDeadlineSeconds"] <= 900 + assert any(v.get("configMap", {}).get("name") == cm["metadata"]["name"] + for v in job["spec"]["template"]["spec"]["volumes"]) + + +@pytest.mark.parametrize("modes,minutes,task_minutes,pipeline", [ + ("{vm}", 60, 12, "1h30m"), ("{vm,container}", 1, 12, "1h30m0s"), + ("{vm,container}", 30, 32, "1h40m0s"), ("{vm,container}", 60, 62, "2h40m0s"), +]) +def test_container_enrollment_budget_reaches_task_form_and_pipeline(modes, minutes, task_minutes, pipeline): + docs = render("--set", f"portal.deploymentModes={modes}", "--set", f"portal.sandboxReadyTimeoutMinutes={minutes}") + tasks = one(docs, "Pipeline", "saw-workspace-create")["spec"]["tasks"] + task = next(t for t in tasks if t["name"] == "vm") + assert task["timeout"] == (f"{task_minutes}m0s" if task_minutes < 60 else "1h2m0s") + assert next(p["value"] for p in task["params"] if p["name"] == "timeout") == "600" + cm = one(docs, "ConfigMap", "saw-rhdh-templates") + for name in ("create-workspace.yaml", "create-workspace-for-user.yaml"): + if name not in cm["data"]: + continue + steps = {s["id"]: s for s in yaml.safe_load(cm["data"][name])["spec"]["steps"]} + assert len(steps["task-vm"]["each"]) == 3 * task_minutes + assert steps["run"]["input"]["body"]["spec"]["timeouts"]["pipeline"] == pipeline + + +def test_portal_allows_cold_plugin_and_backend_startup(docs): + spec = one(docs, "Backstage", "developer-hub")["spec"]["deployment"]["patch"]["spec"] + assert spec["progressDeadlineSeconds"] == 1800 + backend = next(c for c in spec["template"]["spec"]["containers"] if c["name"] == "backstage-backend") + assert backend["startupProbe"] == {"failureThreshold": 15} + + +@pytest.mark.ci_external +def test_kata_forms_offer_only_profiles_approved_for_that_placement(): + uid = "11111111-1111-1111-1111-111111111111" + sites = { + "alice": {"subject": uid, "automaticEnrollment": {"approvedProfiles": {"custom-inference": "1"*64}}}, + "bob": {"subject": "22222222-2222-2222-2222-222222222222", + "kataRuntimeClass": {"name": "site-kata", "uid": uid, "sha256": "2"*64}, + "automaticEnrollment": {"approvedProfiles": {"data-science": "3"*64}}}, + } + docs = render("--set-json", 'portal.deploymentModes=["vm","container","kata"]', + "--set-json", 'portal.admins=["admin"]', "--set-json", "portal.containerSites="+json.dumps(sites)) + templates = one(docs, "ConfigMap", "saw-rhdh-templates")["data"] + for name in ("create-workspace.yaml", "create-workspace-for-user.yaml"): + step, fields = _deployment_profiles(yaml.safe_load(templates[name])) + assert step["properties"]["deploymentMode"]["enumNames"] == ["Virtual machine", "Container", "Kata"] + assert fields["container"]["enum"] == ["custom-inference"] + assert fields["kata"]["enum"] == ["data-science"] + assert "site-kata" not in templates[name] diff --git a/tests/charts/test_openshell_saw_chart.py b/tests/charts/test_openshell_saw_chart.py index 0d5f963c4..f5354222e 100644 --- a/tests/charts/test_openshell_saw_chart.py +++ b/tests/charts/test_openshell_saw_chart.py @@ -799,7 +799,15 @@ def test_rendered_proxies_verify_issuer_tls_by_default(settings, skip): def test_installer_configmap_ships_the_real_files(default_docs, ab): data = installer_data(default_docs) - assert data["apply_bom.py"] == (CHART / "files" / "installer" / "apply_bom.py").read_text() + source = (CHART / "files" / "installer" / "apply_bom.py").read_text() + for name in ("core", "admin", "ui_limit"): + shared = (CHART / "files/installer" / ("saw_"+name+".py")).read_bytes() + field = "_SAW_"+name.upper()+"_EMBEDDED_B64" + marker = field+' = "__SAW_'+name.upper()+'_SOURCE_BASE64__"' + assert source.count(marker) == 1 + source = source.replace(marker, field+' = "'+base64.b64encode(shared).decode("ascii")+'"') + assert "saw_"+name+".py" not in data + assert data["apply_bom.py"] == source assert data["setup-dashboard.sh"] == (CHART / "files" / "installer" / "setup-dashboard.sh").read_text() bom = yaml.safe_load(data["installer-bom.yaml"]) assert ab.validate_bom(bom) @@ -1564,12 +1572,49 @@ def test_the_vm_runs_on_arm64_too(default_docs): assert "smm" not in domain["features"] +# -- deployment backend contract ---------------------------------------------- + +@pytest.mark.ci_external +def test_explicit_vm_mode_preserves_default_resources(): + assert render() == render("--set", "deployment.mode=vm") + + +@pytest.mark.ci_external +@pytest.mark.parametrize("mode", ["container", "kata", "confidential"]) +def test_unimplemented_backend_never_falls_back_to_vm(mode): + result = helm_template(CHART, "--set", "sandboxName=saw-test", + "--set", f"deployment.mode={mode}") + assert result.returncode != 0 + if mode == "container": + assert "deployment.mode=container" in result.stderr and "no VM fallback" in result.stderr + else: + assert "deployment.mode" in result.stderr or "/deployment/mode" in result.stderr + assert not result.stdout.strip() + + +@pytest.mark.ci_external +@pytest.mark.parametrize("skip_schema", [False, True]) +@pytest.mark.parametrize("deployment", [ + {"mode": "typo"}, {"mode": ""}, {"mode": False}, {"mode": 1}, + {"mode": None}, {"mode": []}, {"mod": "container"}, "container", [], None, +]) +def test_invalid_deployment_contract_rejected(deployment, skip_schema): + flags = ["--skip-schema-validation"] if skip_schema else [] + assert "deployment" in render_error( + *flags, "--set-json", f"deployment={json.dumps(deployment)}") + + +@pytest.mark.ci_external +@pytest.mark.parametrize("mode", ["container", "kata", "confidential", "typo"]) +def test_mode_guard_also_applies_when_schema_validation_is_disabled(mode): + assert "deployment.mode" in render_error( + "--skip-schema-validation", "--set", f"deployment.mode={mode}") + + @pytest.mark.ci_external @pytest.mark.parametrize("override,expected", [(None, 180), (360, 360)]) def test_vm_shutdown_grace_exceeds_guest_service_stop_timeout(override, expected): args = () if override is None else ("--set", f"vm.terminationGracePeriodSeconds={override}") vm = render(*args)[("VirtualMachine", "saw-test")] assert vm["spec"]["template"]["spec"]["terminationGracePeriodSeconds"] == expected - # The configurable grace belongs to the VM instance, rather than a pod - # annotation or domain field that KubeVirt would ignore. assert "terminationGracePeriodSeconds" not in vm["spec"]["template"]["spec"]["domain"] diff --git a/tests/charts/test_portal_container.py b/tests/charts/test_portal_container.py new file mode 100644 index 000000000..6ffeb3a1f --- /dev/null +++ b/tests/charts/test_portal_container.py @@ -0,0 +1,416 @@ +"""Render the same owner request through both Helm levels and the stock component charts.""" +import importlib.util +from pathlib import Path +import json +import subprocess + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +U = "alice" +REG = "a8cb6cbd0eb62f31" +SITE = yaml.safe_load((ROOT / "tests/fixtures/container-site.yaml").read_text())["portal"]["containerSites"][U] +GLOBAL = {"repoURL":"https://github.com/example/secure-agent-workspace","targetRevision":"test", + "vpArgoNamespace":"vp-gitops","clusterDomain":"cluster.example.com"} + + +def render(chart, release, ns, values): + result = subprocess.run(["helm","template",release,str(ROOT/"charts"/chart),"--namespace",ns, + "--api-versions","agents.x-k8s.io/v1beta1","--api-versions","policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy", + "--api-versions","admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy","-f","-"], + input=json.dumps(values),text=True,capture_output=True) + assert result.returncode == 0, result.stderr + return [d for d in yaml.safe_load_all(result.stdout) if d] + + +def child(app): + src = app["spec"]["source"] + helm = src["helm"] + return render(src["path"].split("/")[-1],helm["releaseName"],app["spec"]["destination"]["namespace"], + helm.get("valuesObject") or yaml.safe_load(helm["values"])) + + +def user(): + return {"name":U, "ownerSubject":SITE["subject"],"installationID":"802fe583-9329-54b6-b5b4-7be58ec171f3", + "registration":REG,"intentGeneration":1,"vaultPrefix":f"secret/data/hub/saw-{U}/{REG}", + "values":{"deployment":{"mode":"container"}},"profiles":["data-science"],"pruneOnRemove":True} + + +def test_container_has_own_gateway_credential_audit_and_policy(): + top = render("saw-users","portal-ws-"+U,"saw-portal",{"global":GLOBAL,"users":[user()],"containerSites":{U:SITE}}) + apps = [a for a in top if a["kind"]=="Application"] + assert {a["spec"]["source"]["path"] for a in apps} == {"charts/saw-bom","charts/saw-portal-container"} + resources = [r for a in apps for r in child(a)] + assert all(isinstance(r["spec"]["source"]["targetRevision"], str) + for r in resources if r["kind"] == "Application") + es, = [r for r in resources if r["kind"]=="ExternalSecret" and r["metadata"]["name"]=="inference"] + assert es["metadata"]["namespace"] == "saw-ctl-"+REG + assert es["spec"]["data"] == [{"secretKey":"api_key","remoteRef":{"key":user()["vaultPrefix"]+"/inference","property":"api_key"}}] + archived_ns, = [r for r in resources if r["kind"]=="Namespace" and r["metadata"]["name"]=="saw-history-"+REG] + archive_annotations=archived_ns["metadata"]["annotations"] + assert archive_annotations["argocd.argoproj.io/compare-options"]=="IgnoreExtraneous" + assert set(archive_annotations["argocd.argoproj.io/sync-options"].split(","))=={"Delete=false","Prune=false"} + workloads = [r for a in resources if a["kind"]=="Application" for r in child(a)] + assert not any(r["kind"]=="VirtualMachine" for r in workloads) + net, = [r for r in workloads if r["kind"]=="AdminNetworkPolicy"] + assert net["spec"]["subject"]["pods"]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"] == "saw-"+U + assert any(r["kind"]=="StatefulSet" and r["metadata"]["name"] == U+"-gateway" for r in workloads) + history, = [r for r in workloads if r["kind"]=="PersistentVolumeClaim" and r["metadata"]["name"].endswith("-history")] + assert history["metadata"]["namespace"] == "saw-history-"+REG + assert "Delete=false" in history["metadata"]["annotations"]["argocd.argoproj.io/sync-options"] + assert "saw-unrelated-vm" not in json.dumps(workloads) + + +def test_container_rejects_another_registrations_keys(): + v = dict(owner=U,ownerSubject=SITE["subject"],registration=REG,vaultPrefix="secret/data/hub/saw-other/"+REG) + with pytest.raises(AssertionError,match="vaultPrefix must match"): + render("saw-portal-container",U,"saw-"+U,v) + + +def test_automatic_enrollment_cannot_read_secrets_or_mount_history(): + site=SITE + v=dict(owner=U,ownerSubject=site["subject"],registration=REG,installationID=user()["installationID"],vaultPrefix=user()["vaultPrefix"],site=site,**{"global":GLOBAL}) + resources=render("saw-portal-container",U,"saw-"+U,v) + j,=[r for r in resources if r["kind"]=="Job"] + spec=j["spec"]["template"]["spec"] + assert not any("persistentVolumeClaim" in v for v in spec["volumes"]) + assert not any("secret" in v for v in spec["volumes"]) + assert "gateway-ca" not in {v["name"] for v in spec["volumes"]} + assert all(mount["name"] != "gateway-ca" + for container in spec["containers"] for mount in container.get("volumeMounts", [])) + control_role, = [r for r in resources if r["kind"] == "Role" + and r["metadata"]["namespace"] == "saw-ctl-"+REG + and r["metadata"]["name"] == "enroll-"+REG] + ca_rule, = [rule for rule in control_role["rules"] + if "configmaps" in rule["resources"] and U+"-gateway-ca" in rule.get("resourceNames", [])] + assert ca_rule["verbs"] == ["get"] + assert U+"-gateway-ca" in ca_rule["resourceNames"] + workspace_role, = [r for r in resources if r["kind"] == "Role" + and r["metadata"]["namespace"] == "saw-"+U + and r["metadata"]["name"] == "enroll-"+REG] + assert all(U+"-gateway-ca" not in rule.get("resourceNames", []) + for rule in workspace_role["rules"]) + script_volume, = [volume for volume in spec["volumes"] if volume["name"] == "scripts"] + script_config, = [resource for resource in resources + if resource["kind"] == "ConfigMap" + and resource["metadata"]["name"] == script_volume["configMap"]["name"]] + assert {"http_errors.py", "kube_api.py"} <= set(script_config["data"]) + grants=[rule for r in resources if r["kind"] in ("Role","ClusterRole") and "enroll" in r["metadata"]["name"] for rule in r["rules"]] + assert all("secrets" not in x["resources"] and "*" not in x["verbs"] for x in grants) + assert [x["resourceNames"] for x in grants if "patch" in x["verbs"]]==[["saw-portal-enrollment"]] + files=ROOT/"charts/saw-portal-container/files" + assert (files/"receipt.py").read_bytes()==(ROOT/"kubernetes-runtime/saw_kubernetes/receipt.py").read_bytes() + assert (files/"gateway_ca.py").read_bytes()==(ROOT/"kubernetes-runtime/saw_kubernetes/gateway_ca.py").read_bytes() + assert (files/"portal_contract.py").read_text()==(ROOT/"kubernetes-runtime/saw_kubernetes/portal_contract.py").read_text().replace("from .boundary import","from saw_kubernetes.boundary import") + + +def test_runtime_controller_reads_only_the_named_public_gateway_ca(): + values = yaml.safe_load((ROOT / "tests/fixtures/saw-kubernetes-controller-values.yaml").read_text()) + docs = render("saw-kubernetes-controller", "alice", "saw-ci-trusted", values) + role, = [r for r in docs if r["kind"] == "Role" and r["metadata"]["namespace"] == "saw-ci-trusted"] + ca_rule, = [r for r in role["rules"] if "configmaps" in r["resources"] and "alice-gateway-ca" in r["resourceNames"]] + assert ca_rule["verbs"] == ["get"] + workspace_role, = [r for r in docs if r["kind"] == "Role" and r["metadata"]["namespace"] == "saw-alice"] + assert all("alice-gateway-ca" not in r.get("resourceNames", []) for r in workspace_role["rules"]) + deployment, = [r for r in docs if r["kind"] == "Deployment"] + pod = deployment["spec"]["template"]["spec"] + assert "identity" not in {v["name"] for v in pod["volumes"]} + assert all(m["name"] != "identity" for c in pod["containers"] for m in c["volumeMounts"]) + + +def test_status_reader_keeps_rhdh_and_provisioner_access_with_static_observer(): + site = {**SITE, "observer": {"namespace": "saw-workspaces", "serviceAccount": "saw-projector-owner"}} + docs = render("saw-portal-container", U, "saw-"+U, dict( + site=site, owner=U, registration=REG, installationID=user()["installationID"], + ownerSubject=SITE["subject"], vaultPrefix=user()["vaultPrefix"], **{"global": GLOBAL})) + binding, = [o for o in docs if o["kind"] == "RoleBinding" + and o["metadata"]["name"] == "saw-portal-runtime-observer"] + assert {(s["namespace"], s["name"]) for s in binding["subjects"]} == { + ("saw-workspaces", "saw-projector-owner"), + ("saw-portal", "saw-workspaces-generator"), + ("saw-portal", "saw-portal-provisioner"), + } + role, = [o for o in docs if o["kind"] == "Role" + and o["metadata"]["name"] == "saw-portal-runtime-observer"] + assert role["rules"] == [{"apiGroups": [""], "resources": ["configmaps"], + "resourceNames": ["saw-bom-profiles", "saw-runtime-intent", "saw-runtime-status"], + "verbs": ["get"]}] + + +def test_multiple_catalog_sandboxes_receive_distinct_storage_and_ui_namespaces(): + import importlib.util + spec = importlib.util.spec_from_file_location("portal_topology",ROOT/"charts/openshell-rhdh/files/portal.py") + portal=importlib.util.module_from_spec(spec) + spec.loader.exec_module(portal) + catalog=json.loads((ROOT/"charts/saw-users/files/profile-catalog.json").read_text())["profiles"] + targets=portal.sandbox_targets(U,REG,catalog["data-science"]) + assert set(targets)=={"default/notebook","cuda-dev/cuda-sandbox"} + site={**SITE,"targets":targets,"selectedDigest":"a"*64} + docs=render("saw-portal-container",U,"saw-"+U,dict(site=site,owner=U,registration=REG,installationID=user()["installationID"],ownerSubject=SITE["subject"],vaultPrefix=user()["vaultPrefix"],**{"global":GLOBAL})) + writers=[o for o in docs if o["kind"]=="Application" and o["spec"]["source"]["path"]=="charts/saw-audit-writer"] + assert len(writers)==2 + assert {o["spec"]["source"]["helm"]["releaseName"] for o in writers}=={x["base"] for x in targets.values()} + assert {o["metadata"]["name"] for o in docs if o["kind"]=="Namespace" and o["metadata"]["name"].startswith("saw-ui-")}=={"saw-ui-"+x["relayID"] for x in targets.values()} + for writer in writers: + identity=writer["spec"]["source"]["helm"]["valuesObject"]["identity"] + assert identity["workspace"]+"/"+identity["sandbox"] in targets + + +@pytest.mark.parametrize("older_component", [None, "identity", "gateway"]) +def test_gateway_api_versions_require_the_same_reviewed_identity_revision(older_component): + site = json.loads(json.dumps(SITE)) + revision = "a" * 40 + site["stack"]["identity"]["revision"] = revision + site["stack"]["gateway"]["revision"] = revision + if older_component: + site["stack"][older_component]["revision"] = "b" * 40 + values = dict(site=site, owner=U, registration=REG, installationID=user()["installationID"], + ownerSubject=SITE["subject"], vaultPrefix=user()["vaultPrefix"], + **{"global": {**GLOBAL, "targetRevision": revision}}) + apps = [r for r in render("saw-portal-container", U, "saw-" + U, values) + if r["kind"] == "Application"] + gateway, = [app for app in apps if app["spec"]["source"]["path"] == "charts/openshell-saw-kubernetes"] + expected = ["agents.x-k8s.io/v1beta1/Sandbox", "policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy"] + for app in apps: + helm = app["spec"]["source"]["helm"] + if app is gateway and older_component is None: + assert helm["apiVersions"] == expected + else: + assert "apiVersions" not in helm + + +def test_model_gate_claim_is_shared_only_for_the_publisher_and_agent(): + audit = SITE["stack"]["audit"]["values"] + # Use the same per-registration Helm values projection as the nested Applications. + target = dict(workspace="default", sandbox="notebook", ui=True, base=U, + relayID=REG, subjectMap="saw-portal-relay-subject", modelInput="inference.model") + site = {**SITE, "lifecycleVersion": 1, "targets": {"default/notebook": target}} + manifests = render("saw-portal-container", U, "saw-"+U, { + "site": site, "owner": U, "registration": REG, "installationID": user()["installationID"], + "ownerSubject": SITE["subject"], "vaultPrefix": user()["vaultPrefix"], "global": GLOBAL}) + writer, = [r for r in manifests if r["kind"]=="Application" and r["spec"]["source"]["path"]=="charts/saw-audit-writer"] + claims = {r["metadata"]["name"]:r["spec"] for r in child(writer) if r["kind"]=="PersistentVolumeClaim"} + assert claims["saw-gate-"+U]["accessModes"] == ["ReadWriteMany"] + assert claims["saw-gate-"+U]["storageClassName"] == audit["exchangeStorageClass"] + assert claims["workspace-"+U]["accessModes"] == claims[U+"-history"]["accessModes"] == ["ReadWriteOnce"] + + +def test_personal_assistant_uses_the_vm_proxy_policy_and_trusted_secret_projection(): + owner = {**user(), "profiles":["personal-assistant"]} + site = {**SITE, "providerAllowHosts":["integrate.api.nvidia.com","slack.com","gmail.googleapis.com","oauth2.googleapis.com"]} + site["automaticEnrollment"]={**SITE["automaticEnrollment"],"approvedProfiles":{"personal-assistant":"a"*64}} + parent = render("saw-users","portal-ws-"+U,"saw-portal",{"global":GLOBAL,"users":[owner],"containerSites":{U:site}}) + app, = [a for a in parent if a.get("spec",{}).get("source",{}).get("path")=="charts/saw-portal-container"] + resources=child(app) + broker,=[r for r in resources if r["kind"]=="ConfigMap" and r["metadata"]["name"].endswith("-proxy-policy")] + policy=json.loads(broker["data"]["config.json"]) + bindings={b["secret"]:b for b in policy["bindings"]} + assert set(bindings)=={"gmail","slack","inference"} + assert bindings["gmail"]["refresh"]["tokenURL"]=="https://oauth2.googleapis.com/token" + assert bindings["slack"]=={"secret":"slack","key":"bot_token","providers":["slack"]} + secrets={r["metadata"]["name"]:r for r in resources if r["kind"]=="ExternalSecret"} + for name,keys in {"gmail":{"refresh_token","client_id","client_secret"},"slack":{"bot_token"},"inference":{"api_key"}}.items(): + assert secrets[name]["metadata"]["namespace"]=="saw-ctl-"+REG + assert {e["secretKey"] for e in secrets[name]["spec"]["data"]}==keys + proxy,=[r for r in resources if r["kind"]=="Application" and r["spec"]["source"]["path"]=="charts/saw-kubernetes-proxy"] + assert proxy["spec"]["source"]["helm"]["valuesObject"]["credentialSecretNames"]==sorted(secrets) + + +def test_custom_inference_projection_pins_broker_url_and_keeps_key_private(): + url="https://models.fixture.test/v1" + owner={**user(),"profiles":["custom-inference"],"webSearch":{"enabled":False}} + site={**SITE,"providerAllowHosts":["models.fixture.test"],"customInferenceURL":url, + "automaticEnrollment":{**SITE["automaticEnrollment"],"approvedProfiles":{"custom-inference":"a"*64}}} + root=render("saw-users","portal-ws-"+U,"saw-portal",{"global":GLOBAL,"users":[owner],"containerSites":{U:site}}) + app,=[a for a in root if a.get("spec",{}).get("source",{}).get("path")=="charts/saw-portal-container"] + resources=child(app) + broker,=[r for r in resources if r["kind"]=="ConfigMap" and r["metadata"]["name"].endswith("-proxy-policy")] + policy=json.loads(broker["data"]["config.json"]) + assert policy["allowHosts"]==["models.fixture.test"] and not policy["profiles"].get("custom-inference") + assert policy["bindings"]==[{"providers":["custom-inference"],"key":"api_key","secret":"inference"}] + es,=[r for r in resources if r["kind"]=="ExternalSecret"] + assert es["metadata"]["namespace"]=="saw-ctl-"+REG + assert es["spec"]["target"]["template"]=={"engineVersion":"v2","mergePolicy":"Merge","data":{"url":url}} + assert [e["secretKey"] for e in es["spec"]["data"]]==["api_key"] + + +def test_enrollment_job_uses_immutable_revisioned_inputs(): + from copy import deepcopy + values = dict(owner=U, ownerSubject=SITE["subject"], registration=REG, + installationID=user()["installationID"], vaultPrefix=user()["vaultPrefix"], + site=deepcopy(SITE), **{"global": GLOBAL}) + def publication(v): + resources = render("saw-portal-container", U, "saw-"+U, v) + job, = [o for o in resources if o["kind"] == "Job"] + configs = {o["metadata"]["name"]: o for o in resources if o["kind"] == "ConfigMap"} + volume, = [v for v in job["spec"]["template"]["spec"]["volumes"] if v["name"] == "scripts"] + source = configs[volume["configMap"]["name"]] + assert source["immutable"] is True + return job["metadata"]["name"], source["metadata"]["name"] + initial = publication(values) + assert publication(values) == initial + values["site"]["automaticEnrollment"]["image"] = "registry.example.test/controller@sha256:" + "b"*64 + changed = publication(values) + assert all(a != b for a, b in zip(initial, changed)) + values["site"]["automaticEnrollment"]["resources"]["limits"]["memory"] = "8Gi" + assert all(a != b for a, b in zip(changed, publication(values))) + + +@pytest.mark.parametrize("override, expected", [ + (None, "0.1.6-rhaiv.4"), + ("0.1.7-rhaiv.1", "0.1.7-rhaiv.1"), +]) +def test_enrollment_compiles_the_selected_gateway_version(override, expected): + from copy import deepcopy + site = deepcopy(SITE) + if override: + site["stack"]["gateway"]["values"]["kubernetes"]["gateway"] = {"version": override} + values = dict(owner=U, ownerSubject=SITE["subject"], registration=REG, + installationID=user()["installationID"], vaultPrefix=user()["vaultPrefix"], + site=site, **{"global": GLOBAL}) + resources = render("saw-portal-container", U, "saw-" + U, values) + job, = [o for o in resources if o["kind"] == "Job"] + script_volume, = [v for v in job["spec"]["template"]["spec"]["volumes"] + if v["name"] == "scripts"] + config, = [o for o in resources if o["kind"] == "ConfigMap" + and o["metadata"]["name"] == script_volume["configMap"]["name"]] + bootstrap = json.loads(config["data"]["config.json"]) + assert bootstrap["gatewayVersion"] == expected + + +def test_enrollment_default_gateway_version_matches_gateway_chart(): + portal_values = yaml.safe_load((ROOT / "charts/saw-portal-container/values.yaml").read_text()) + gateway_values = yaml.safe_load((ROOT / "charts/openshell-saw-kubernetes/values.yaml").read_text()) + gateway_chart = yaml.safe_load((ROOT / "charts/openshell-saw-kubernetes/Chart.yaml").read_text()) + default = portal_values["driverDefaults"]["gatewayVersion"] + assert default == gateway_values["kubernetes"]["gateway"]["version"] + assert default == gateway_chart["appVersion"] + + +def test_static_projection_owns_no_workload_and_generates_same_composition(): + site = dict(SITE) + values = {"global": GLOBAL, "users": [user()], "containerSites": {U: site}, + "projection": {"enabled": True, "applicationSetControllerLabels": {"app.kubernetes.io/name": "vp-gitops-applicationset-controller"}, "egress": [{"ports": [{"protocol": "TCP", "port": 443}]}]}} + docs = render("saw-users", "workspaces", "saw-installation", values) + assert not any(o["kind"] in ("Namespace", "Application", "VirtualMachine") for o in docs) + appset, = [o for o in docs if o["kind"] == "ApplicationSet"] + deployment, = [o for o in docs if o["kind"] == "Deployment"] + assert appset["spec"]["template"]["spec"]["source"]["path"] == "charts/saw-users" + assert appset["spec"]["template"]["spec"]["source"]["targetRevision"] == "{{ .revision }}" + assert appset["spec"]["template"]["spec"]["source"]["helm"]["values"] == "{{ .workspaceValues }}" + configuration, = [o for o in docs if "workspaces.json" in o.get("data", {})] + pod = deployment["spec"]["template"]["spec"] + config_volume, = [v for v in pod["volumes"] if v["name"] == "configuration"] + assert configuration["kind"] == "ConfigMap" + assert configuration["metadata"]["name"] == config_volume["configMap"]["name"] + account, = [o for o in docs if o["kind"] == "ServiceAccount" + and o["metadata"]["name"] == pod["serviceAccountName"]] + assert "data" not in account + cfg = json.loads(configuration["data"]["workspaces.json"]) + observer = cfg["containerSites"][U]["observer"] + assert observer == {"namespace": "saw-installation", "serviceAccount": deployment["metadata"]["name"]} + assert cfg["users"] == [user()] + assert not any(o["kind"] == "Secret" for o in docs), "plugin identity is generated, never committed in values" + # Only the bootstrap can create its two identity Secrets; the server reads public data. + server = deployment["spec"]["template"]["spec"]["serviceAccountName"] + rules = [o for o in docs if o["kind"] in ("Role", "ClusterRole") and o["metadata"]["name"] == server] + assert all("secrets" not in r["resources"] and r["verbs"] == ["get"] for o in rules for r in o["rules"]) + preflight_role, = [o for o in docs if o["kind"] == "ClusterRole" + and o["metadata"]["name"] == server + "-preflight"] + assert all("secrets" not in rule.get("resources", []) for rule in preflight_role["rules"]) + assert any(rule.get("resourceNames") == ["saw-saw-alice-runtime", "saw-saw-alice-runtime-kata"] + for rule in preflight_role["rules"]) + source, = [o for o in docs if o["kind"] == "ConfigMap" and "projector.py" in o.get("data", {})] + assert source["data"]["preflight.py"] == (ROOT / "charts/saw-portal-container/files/preflight.py").read_text() + container_values = dict(owner=U, ownerSubject=SITE["subject"], registration=REG, + installationID=user()["installationID"], vaultPrefix=user()["vaultPrefix"], + site={**SITE, "observer": observer}, **{"global": GLOBAL}) + resources = render("saw-portal-container", U, "saw-"+U, container_values) + readers = [o for o in resources if o["kind"] == "RoleBinding" and o["metadata"]["name"] == "portal-binding-reader"] + assert readers[0]["subjects"] == [{"kind": "ServiceAccount", "name": server, "namespace": "saw-installation"}] + + +def test_static_kata_projector_can_read_only_approved_platform_metadata(): + entry = user() + entry["values"] = {"deployment": {"mode": "kata"}} + site = {**SITE, "kataRuntimeClass": {"name": "kata", "uid": "11111111-1111-4111-8111-111111111111", + "sha256": "a" * 64}} + docs = render("saw-users", "workspaces", "saw-installation", { + "global": GLOBAL, "users": [entry], "containerSites": {U: site}, + "projection": {"enabled": True, "applicationSetControllerLabels": {"app": "controller"}, + "egress": [{"ports": [{"protocol": "TCP", "port": 443}]}]}}) + role, = [obj for obj in docs if obj["kind"] == "ClusterRole" + and obj["metadata"]["name"].endswith("-preflight")] + assert any(rule.get("resources") == ["runtimeclasses"] and rule.get("resourceNames") == ["kata"] + for rule in role["rules"]) + assert any(rule.get("resources") == ["nodes"] and rule.get("verbs") == ["list"] + for rule in role["rules"]) + assert all("secrets" not in rule.get("resources", []) for rule in role["rules"]) + + +def test_static_mixed_backends_keep_vm_resolution_in_the_root(): + values = {"global": GLOBAL, "users": [user(), {"name": "bob"}], "containerSites": {U: SITE}, + "ownerResolution": {"enabled": True}, + "projection": {"enabled": True, "applicationSetControllerLabels": {"app.kubernetes.io/name": "vp-gitops-applicationset-controller"}, "egress": [{"ports": [{"protocol": "TCP", "port": 443}]}]}} + docs = render("saw-users", "workspaces", "saw-installation", values) + apps = [o for o in docs if o["kind"] == "Application"] + assert {o["metadata"]["name"] for o in apps} == {"saw-bob", "saw-bob-bom", "saw-bob-secrets"} + resolver = [o for o in docs if o["kind"] == "ConfigMap" and o["metadata"]["name"] == "saw-owner-subject"] + assert [o["metadata"]["namespace"] for o in resolver] == ["saw-bob"] + configuration, = [o for o in docs if "workspaces.json" in o.get("data", {})] + assert json.loads(configuration["data"]["workspaces.json"])["users"] == [user()] + + +def test_foundation_can_be_reviewed_without_placeholder_approval_or_workload(): + from copy import deepcopy + site = deepcopy(SITE) + site["automaticEnrollment"]["enabled"] = False + del site["automaticEnrollment"]["reference"] + site["automaticEnrollment"].pop("approvedProfiles", None) + render("saw-users", "workspaces", "saw-installation", { + "global": GLOBAL, "users": [user()], "containerSites": {U: site}}) + docs = render("saw-portal-container", U, "saw-"+U, dict( + site=site, owner=U, ownerSubject=site["subject"], registration=REG, + installationID=user()["installationID"], vaultPrefix=user()["vaultPrefix"], **{"global": GLOBAL})) + assert not any(o["kind"] == "Job" for o in docs), "no enrollment before review" + assert not any(o["metadata"]["name"] == "saw-runtime-intent" for o in docs) + assert any(o["metadata"]["name"] == "saw-portal-enrollment" for o in docs) + assert any(o["kind"] == "RoleBinding" and o["metadata"]["name"] == "portal-binding-reader" for o in docs) + assert any(o.get("spec", {}).get("source", {}).get("path") == "charts/openshell-saw-kubernetes" for o in docs) + + +@pytest.mark.parametrize("stream", ["saw-kubernetes-runtime", "custom-runtime"]) +def test_runtime_pull_grants_follow_the_configured_image(stream): + from copy import deepcopy + site = deepcopy(SITE) + image = "image-registry.example.test/builds/" + stream + "@sha256:" + "b" * 64 + site["automaticEnrollment"]["image"] = image + site["enrollment"] = dict( + registration=REG, image=image, buildNamespace="builds", revision="1" * 40, + approved={}, intent={"profileRef": {"name": "saw-bom-profiles"}, "bindings": { + "ownerID": user()["installationID"], "ownerSubject": SITE["subject"], + "namespace": "saw-" + U, "namespaceUID": "fixture-uid", "mode": "container", + "bindings": {"default/notebook": { + "workspace": {"claimName": "workspace-" + U}, + "toolGate": {"trusted": {"claimName": "saw-gate-" + U}, + "audit": {"claimName": "saw-audit-" + U}, + "ownerLogin": U, "receiptKey": "fixture-public-key"}, + "browser": {"ownerLogin": U, "origin": "https://alice-default-notebook-ui.apps.cluster.example.com"}}}}}) + resources = render("saw-portal-container", U, "saw-" + U, dict( + site=site, owner=U, ownerSubject=SITE["subject"], registration=REG, + installationID=user()["installationID"], intentGeneration=1, + vaultPrefix=user()["vaultPrefix"], **{"global": GLOBAL})) + grants = [rule for obj in resources if obj["kind"] == "Role" + for rule in obj["rules"] if "imagestreams/layers" in rule["resources"] + and obj["metadata"]["name"] in {"enroll-" + REG + "-pull", "portal-gate-" + REG}] + assert len(grants) == 2 + assert all(rule["resourceNames"] == [stream] and rule["verbs"] == ["get"] for rule in grants) + +def test_removing_last_static_owner_keeps_only_empty_registration_config(): + docs = render("saw-users", "static", "saw-workspaces", { + "global": GLOBAL, "users": [], "projection": {"enabled": True}}) + assert len(docs) == 1 + assert docs[0]["kind"] == "ConfigMap" + assert docs[0]["metadata"]["name"].endswith("-configuration") + assert json.loads(docs[0]["data"]["workspaces.json"]) == {"users": []} diff --git a/tests/charts/test_saw_image_cache.py b/tests/charts/test_saw_image_cache.py new file mode 100644 index 000000000..c870420b8 --- /dev/null +++ b/tests/charts/test_saw_image_cache.py @@ -0,0 +1,45 @@ +from pathlib import Path +import subprocess + +import pytest +import yaml + + +CHART = Path(__file__).resolve().parents[2] / "charts/saw-image-cache" +IMAGE = "example.test/models/cuda@sha256:" + "a" * 64 + + +def render(*args): + return subprocess.run(["helm", "template", "approved-images", str(CHART), *args], + capture_output=True, text=True) + + +def test_cache_is_opt_in(): + result = render() + assert result.returncode == 0 and not result.stdout.strip() + + +@pytest.mark.parametrize("images", [[], ["example.test/models/cuda:latest"], [IMAGE + "BAD"]]) +def test_enabled_cache_rejects_missing_or_mutable_references(images): + args = [arg for index, image in enumerate(images) + for arg in ("--set-string", f"images[{index}]={image}")] + assert render("--set", "enabled=true", *args).returncode != 0 + + +def test_cache_mounts_approved_images_without_executing_them(): + result = render("--set", "enabled=true", "--set-string", f"images[0]={IMAGE}", + "--set-string", f"images[1]={IMAGE}") + assert result.returncode == 0, result.stderr + objects = {o["kind"]: o for o in yaml.safe_load_all(result.stdout)} + assert set(objects) == {"DaemonSet", "ServiceAccount", "NetworkPolicy"} + pod = objects["DaemonSet"]["spec"]["template"]["spec"] + assert not pod["automountServiceAccountToken"] and not objects["ServiceAccount"]["automountServiceAccountToken"] + assert pod["securityContext"] == {"runAsNonRoot": True, "seccompProfile": {"type": "RuntimeDefault"}} + assert pod["volumes"] == [{"name": "image-0", "image": {"reference": IMAGE, "pullPolicy": "IfNotPresent"}}] + holder, = pod["containers"] + assert holder["image"] != IMAGE and holder["command"] == ["/usr/bin/sleep", "infinity"] + assert holder["securityContext"] == {"readOnlyRootFilesystem": True, "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}} + assert holder["volumeMounts"] == [{"name": "image-0", "mountPath": "/images/0", "readOnly": True}] + policy = objects["NetworkPolicy"]["spec"] + assert set(policy["policyTypes"]) == {"Ingress", "Egress"} and not policy.get("egress") and not policy.get("ingress") diff --git a/tests/charts/test_saw_kubernetes_chart.py b/tests/charts/test_saw_kubernetes_chart.py new file mode 100644 index 000000000..1d5e918fa --- /dev/null +++ b/tests/charts/test_saw_kubernetes_chart.py @@ -0,0 +1,269 @@ +"""Stock gateway integration contracts; no claim of cluster enforcement.""" +import json +import shutil +import tomllib +from pathlib import Path + +import pytest +import yaml +from helm_cache import run_helm + +ROOT = Path(__file__).resolve().parents[2] +CHART = ROOT / 'charts/openshell-saw-kubernetes' +HELM = shutil.which('helm') +pytestmark = [pytest.mark.ci_external, pytest.mark.skipif(not HELM, reason='Helm required')] +APIS = ['--api-versions', 'agents.x-k8s.io/v1beta1/Sandbox', + '--api-versions', 'policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy'] + + +def template(*args, apis=True): + return run_helm([HELM, 'template', 'saw-test', str(CHART), '-n', 'saw-test', + '-f', str(ROOT / 'tests/fixtures/saw-kubernetes-values.yaml'), + *(APIS if apis else []), *args], capture_output=True, text=True) + + +def render(*args): + result = template(*args) + assert result.returncode == 0, result.stderr + return {(d['kind'], d['metadata']['name']): d + for d in yaml.safe_load_all(result.stdout) if d} + + +def config(docs): + return tomllib.loads(docs['ConfigMap', 'saw-test-gateway-config']['data']['gateway.toml'])['openshell'] + + +def test_stock_split_topology_and_server_trust(): + docs = render() + cfg = config(docs) + assert cfg['version'] == 2 + assert cfg['gateway']['compute_driver'] == 'kubernetes' + driver = cfg['drivers']['kubernetes'] + assert driver['workspace_mode'] == 'shared' + assert driver['namespace'] == 'saw-test' + assert driver['client_tls_secret_name'] == 'gateway-server' + assert driver['proxy_connect_by_hostname'] is True + assert driver['sandbox_runtime'] == {'boundary_port': 5500} + assert driver['resource_admission']['enabled'] is True + assert driver['resource_admission']['required_labels']['openshell.ai/sandbox-attachable-workspace'] == '${workspace}' + assert 'supervisor_runtime_class_name' not in driver + assert 'supervisor_egress_mode' not in driver + assert 'topology' not in driver + assert not {kind for kind, _ in docs} & {'VirtualMachine', 'DataVolume', 'Sandbox', 'ClusterPolicy'} + + +def test_user_oidc_and_trusted_reconciler_auth_fail_closed(): + cfg = config(render())['gateway'] + assert cfg['auth'] == {'allow_unauthenticated_users': False} + assert cfg['mtls_auth'] == {'enabled': False, 'operator_enabled': True} + assert cfg['oidc']['dangerously_allow_insecure_http'] is False + assert cfg['tls']['client_ca_path'] == '/etc/openshell-tls/client-ca/ca.crt' + assert cfg['policy_validation_failure_mode'] == 'fail_closed' + assert cfg['interceptors'][0]['failure_policy'] == 'fail_closed' + + +def test_gateway_security_mounts_and_persistence(): + docs = render() + sts = docs['StatefulSet', 'saw-test-gateway'] + spec = sts['spec']['template']['spec'] + assert sts['spec']['replicas'] == 1 + assert 'runtimeClassName' not in spec + assert spec['securityContext']['runAsNonRoot'] is True + container = spec['containers'][0] + assert container['securityContext'] == {'allowPrivilegeEscalation': False, + 'readOnlyRootFilesystem': True, + 'capabilities': {'drop': ['ALL']}} + assert container['readinessProbe']['httpGet']['path'] == '/readyz' + mounts = {m['name']: m for m in container['volumeMounts']} + assert mounts['config']['subPath'] == 'gateway.toml' + assert 'checksum/config' in sts['spec']['template']['metadata']['annotations'] + volumes = {v['name']: v for v in spec['volumes']} + assert volumes['client-ca']['secret']['items'] == [{'key': 'ca.crt', 'path': 'ca.crt'}] + assert volumes['proxy-ca']['configMap']['name'] == 'saw-proxy-ca' + names = {v['secret']['secretName'] for v in spec['volumes'] if 'secret' in v} + assert names == {'gateway-server', 'gateway-client-ca', 'gateway-jwt'} + assert not any(kind == 'Secret' for kind, _ in docs) + pvc = docs['PersistentVolumeClaim', 'saw-test-gateway-data'] + assert pvc['metadata']['annotations']['argocd.argoproj.io/sync-options'] == 'Prune=false,Delete=false' + assert not pvc['metadata'].get('ownerReferences') + + +def test_existing_claim_is_never_created_or_owned(): + docs = render('--set', 'kubernetes.storage.existingGatewayClaim=retained-gateway') + assert not any(kind == 'PersistentVolumeClaim' for kind, _ in docs) + volumes = docs['StatefulSet', 'saw-test-gateway']['spec']['template']['spec']['volumes'] + assert next(v for v in volumes if v['name'] == 'data')['persistentVolumeClaim']['claimName'] == 'retained-gateway' + + +def test_rbac_cannot_read_provider_secrets_or_control_other_namespaces(): + docs = render() + role = docs['Role', 'saw-test-gateway'] + secret_reads = [r for r in role['rules'] if 'secrets' in r['resources'] and 'get' in r['verbs']] + assert secret_reads == [{'apiGroups': [''], 'resources': ['secrets'], + 'resourceNames': ['gateway-server'], 'verbs': ['get']}] + assert not any('pods/exec' in r['resources'] or '*' in r['verbs'] for r in role['rules']) + cluster = docs['ClusterRole', 'saw-test-saw-test-gateway'] + assert not any(set(r['resources']) & {'secrets', 'pods', 'sandboxes', 'networkpolicies'} for r in cluster['rules']) + assert all(r.get('resourceNames') == ['saw-test'] for r in cluster['rules'] if 'namespaces' in r['resources']) + assert docs['ServiceAccount', 'saw-test-sandbox']['automountServiceAccountToken'] is False + + +def test_gateway_service_policy_allows_only_paired_internal_services(): + docs = render() + validations = docs['ValidatingAdmissionPolicy', 'saw-test-services-driver']['spec']['validations'] + assert len(validations) == 2 + service_rule = validations[1]['expression'] + for fragment in ( + "'os-boundary-' + variables.id", + "'os-supervisor-' + variables.id", + "variables.obj.metadata.ownerReferences[0].kind == 'Sandbox'", + "variables.obj.spec.type == 'ClusterIP'", + "!has(variables.obj.spec.externalIPs)", + "!has(variables.obj.spec.externalName)", + "variables.obj.spec.selector.size() >= 2", + "variables.obj.spec.selector['openshell.ai/boundary-role'] == 'supervisor'", + "variables.obj.spec.selector['openshell.ai/boundary-pair'] == variables.id", + "variables.obj.spec.selector['openshell.ai/sandbox-id'] == variables.id", + "variables.obj.spec.ports[0].port == 5500", + "variables.obj.spec.publishNotReadyAddresses", + ): + assert fragment in service_rule + assert 'NodePort' not in service_rule and 'LoadBalancer' not in service_rule + + +def test_network_does_not_grant_workload_or_namespace_wide_egress(): + docs = render() + policies = [d for (kind, _), d in docs.items() if kind == 'NetworkPolicy'] + deny = docs['NetworkPolicy', 'saw-test-default-deny']['spec'] + assert deny['egress'] == deny['ingress'] == [] + granting = [p for p in policies if p['spec'].get('egress')] + assert {p['metadata']['name'] for p in granting} == {'saw-test-gateway', 'openshell-sandbox-supervisors'} + assert docs['NetworkPolicy', 'openshell-sandbox-supervisors']['spec']['podSelector'] == { + 'matchLabels': {'openshell.ai/boundary-role': 'supervisor'}} + rules = docs['Role', 'saw-test-gateway']['rules'] + fence, = [r for r in rules if 'networkpolicies' in r['resources']] + assert fence['verbs'] == ['get', 'create'] + assert 'resourceNames' not in fence + assert not any(p['spec']['podSelector'].get('matchLabels', {}).get('openshell.ai/boundary-role') == 'workload' for p in granting) + assert docs['Route', 'saw-test-gateway']['spec']['tls'] == {'termination': 'passthrough'} + + +def test_repeat_renders_have_no_random_keys_or_identity(): + first = template() + second = template() + assert first.returncode == second.returncode == 0 + assert first.stdout == second.stdout + assert 'PRIVATE KEY' not in first.stdout + + +@pytest.mark.parametrize('assignment', [ + 'kubernetes.experimental=false', 'deployment.mode=confidential', + 'deployment.mode=vm', 'kubernetes.gateway.image=repo:latest', + 'kubernetes.identity.serverTlsSecret=', 'kubernetes.storage.className=', + 'kubernetes.workload.runtimeClassName=kata', 'kubernetes.supervisor.runtimeClassName=kata', + 'oidc.issuerUrl=http://issuer.test', 'kubernetes.proxy.caConfigMap=', + 'kubernetes.governance.endpoint=http://outside.example.test:18081', + 'kubernetes.network.reconcilerNamespace=saw-test', +]) +def test_invalid_or_unimplemented_inputs_fail_before_apply(assignment): + result = template('--set', assignment) + assert result.returncode != 0 + assert not result.stdout.strip() + + +def test_unsupported_backend_and_missing_dependencies_fail(): + for args, apis in [(['--set', 'deployment.mode=kata'], True), ([], False)]: + result = template(*args, apis=apis) + assert result.returncode != 0 + assert not result.stdout.strip() + + +@pytest.mark.parametrize('field,url', [ + ('kubernetes.proxy.url', 'http://user:example-secret@proxy.test:3128'), + ('kubernetes.proxy.url', 'http://proxy.test:3128?token=example-secret:1'), + ('kubernetes.governance.endpoint', 'https://user:example-secret@gov.test:443'), + ('oidc.issuerUrl', 'https://user:example-secret@issuer.test/realm'), + ('oidc.issuerUrl', 'https://issuer.test/realm?token=example-secret'), +]) +def test_credentials_cannot_enter_configmap_through_urls(field, url): + result = template('--set-string', f'{field}={url}') + assert result.returncode != 0 + assert not result.stdout.strip() + assert 'example-secret' not in result.stderr + + +@pytest.mark.parametrize('namespace', ['default', 'kube-system', 'kube-public', + 'openshift-monitoring', 'x' * 51]) +def test_namespace_wide_deny_cannot_target_platform_namespace(namespace): + result = template('--namespace', namespace) + assert result.returncode != 0 + assert not result.stdout.strip() + + +@pytest.mark.parametrize('rules', [[], [{}], [{'to': [{}], 'ports': [{'port': 443}]}], + [{'to': [{'namespaceSelector': {}}], 'ports': [{'port': 443}]}], + [{'to': [{'namespaceSelector': {'matchLabels': {}}}], 'ports': [{'port': 443}]}], + [{'to': [{'namespaceSelector': {'matchExpressions': []}}], 'ports': [{'port': 443}]}], + [{'to': [{'ipBlock': {'cidr': '0.0.0.0/0'}}], 'ports': [{'port': 443}]}]]) +def test_unbounded_gateway_egress_rejected(rules): + result = template('--set-json', 'kubernetes.network.gatewayEgress=' + json.dumps(rules)) + assert result.returncode != 0 + assert not result.stdout.strip() + + +def test_kata_uses_stock_workload_runtime_default_and_named_read(): + docs = render('--set', 'deployment.mode=kata', '--set', 'kubernetes.workload.runtimeClassName=saw-kata') + driver = config(docs)['drivers']['kubernetes'] + assert driver['default_runtime_class_name'] == 'saw-kata' + assert 'supervisor_runtime_class_name' not in driver + rules = docs['ClusterRole', 'saw-test-saw-test-gateway']['rules'] + assert [r for r in rules if 'runtimeclasses' in r['resources']] == [ + {'apiGroups': ['node.k8s.io'], 'resources': ['runtimeclasses'], 'verbs': ['get'], + 'resourceNames': ['saw-kata']}] + + +def test_container_keeps_stock_openshell_startup_timeouts(): + docs = render() + cfg = config(docs) + assert 'provisioning_repair_window_seconds' not in cfg['gateway'] + driver = cfg['drivers']['kubernetes'] + assert 'runtime_bootstrap_grace_seconds' not in driver + assert 'sandbox_attach_request_timeout_seconds' not in driver + container = docs['StatefulSet', 'saw-test-gateway']['spec']['template']['spec']['containers'][0] + env = {item['name']: item.get('value') for item in container['env']} + assert not { + 'OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS', + 'OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS', + 'OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS', + } & set(env) + + +def test_kata_uses_configurable_longer_openshell_startup_timeouts(): + docs = render( + '--set', 'deployment.mode=kata', + '--set', 'kubernetes.workload.runtimeClassName=saw-kata', + '--set', 'kubernetes.startupTimeouts.provisioningRepairWindowSeconds=1200', + '--set', 'kubernetes.startupTimeouts.runtimeBootstrapGraceSeconds=1200', + '--set', 'kubernetes.startupTimeouts.sandboxAttachRequestTimeoutSeconds=1200', + ) + cfg = config(docs) + assert 'provisioning_repair_window_seconds' not in cfg['gateway'] + driver = cfg['drivers']['kubernetes'] + assert 'runtime_bootstrap_grace_seconds' not in driver + assert 'sandbox_attach_request_timeout_seconds' not in driver + container = docs['StatefulSet', 'saw-test-gateway']['spec']['template']['spec']['containers'][0] + env = {item['name']: item.get('value') for item in container['env']} + assert env['OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS'] == '1200' + assert env['OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS'] == '1200' + assert env['OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS'] == '1200' + + +@pytest.mark.parametrize('assignment', [ + 'kubernetes.startupTimeouts.provisioningRepairWindowSeconds=0', + 'kubernetes.startupTimeouts.runtimeBootstrapGraceSeconds=86401', + 'kubernetes.startupTimeouts.sandboxAttachRequestTimeoutSeconds=-1', +]) +def test_openshell_startup_timeouts_must_be_in_supported_range(assignment): + result = template('--set', assignment) + assert result.returncode != 0 + assert not result.stdout.strip() diff --git a/tests/charts/test_saw_kubernetes_identity_chart.py b/tests/charts/test_saw_kubernetes_identity_chart.py new file mode 100644 index 000000000..49d0e962c --- /dev/null +++ b/tests/charts/test_saw_kubernetes_identity_chart.py @@ -0,0 +1,71 @@ +import json +from pathlib import Path +import shutil + +import pytest +import yaml +from helm_cache import run_helm + +ROOT = Path(__file__).resolve().parents[2] +HELM = shutil.which("helm") +pytestmark = [pytest.mark.ci_external, pytest.mark.skipif(not HELM, reason="Helm required")] + + +def render(*args, enabled=True): + return run_helm([HELM, "template", "identity", str(ROOT / "charts/saw-kubernetes-identity"), + "-n", "saw-test-trusted", "--set", "enabled=" + str(enabled).lower(), + "--set", "gateway=saw-test,targetNamespace=saw-test,ownerID=67a9891d-e281-442f-86c3-41771f5ce049", + "--set", "serverSans[0]=saw-test-gateway.saw-test.svc", + "--set", "serverSans[1]=saw-test-gateway.saw-test.svc.cluster.local", *args], + capture_output=True, text=True) + + +def test_stock_generator_and_restricted_projection_job(): + result = render() + assert result.returncode == 0, result.stderr + assert render().stdout == result.stdout + docs = [d for d in yaml.safe_load_all(result.stdout) if d] + assert not any(d["kind"] == "Secret" for d in docs) + job = next(d for d in docs if d["kind"] == "Job") + pod = job["spec"]["template"]["spec"] + assert pod["automountServiceAccountToken"] is False + assert pod["securityContext"] == {"runAsNonRoot": True, "seccompProfile": {"type": "RuntimeDefault"}} + init = pod["initContainers"][0] + assert init["args"][:3] == ["generate-certs", "--output-dir", "/generated/pki"] + assert not any(v["name"] == "api" for v in init["volumeMounts"]) + for c in [init, *pod["containers"]]: + assert "@sha256:" in c["image"] + assert c["securityContext"]["capabilities"] == {"drop": ["ALL"]} + assert c["securityContext"]["allowPrivilegeEscalation"] is False + volume = next(v for v in pod["volumes"] if v["name"] == "generated") + assert volume["emptyDir"] == {"medium": "Memory", "sizeLimit": "16Mi"} + cm = next(d for d in docs if d["kind"] == "ConfigMap") + config = json.loads(cm["data"]["config.json"]) + assert config["namespace"] != config["targetNamespace"] + assert cm["data"]["bootstrap.py"].rstrip() == (ROOT / "charts/saw-kubernetes-identity/files/bootstrap.py").read_text().rstrip() + + +def test_limited_get_create_permissions_and_no_secret_updates(): + docs = list(yaml.safe_load_all(render().stdout)) + for role in [d for d in docs if d["kind"] in {"Role", "ClusterRole"}]: + for rule in role["rules"]: + assert set(rule["verbs"]) <= {"get", "create"} + if "get" in rule["verbs"]: + assert rule["resourceNames"] + trusted = next(d for d in docs if d["kind"] == "Role" and d["metadata"]["namespace"] == "saw-test-trusted") + assert trusted["rules"][1]["resourceNames"] == ["saw-test-identity"] + runtime = next(d for d in docs if d["kind"] == "Role" and d["metadata"]["namespace"] == "saw-test") + assert not any("admin" in n or "identity" in n for n in runtime["rules"][1]["resourceNames"]) + + +def test_disabled_chart_has_no_resources(): + result = render(enabled=False) + assert result.returncode == 0 and not result.stdout.strip() + + +@pytest.mark.parametrize("value", ["targetNamespace=saw-test-trusted", "targetNamespace=kube-system", + "ownerID=", "gatewayImage=repo:latest", "helperImage=repo:latest", "adminDays=0", "adminDays=91", + "serverSans[0]=other.example.test"]) +def test_bad_inputs_fail_render(value): + result = render("--set", value) + assert result.returncode != 0 and not result.stdout.strip() diff --git a/tests/charts/test_saw_kubernetes_proxy_chart.py b/tests/charts/test_saw_kubernetes_proxy_chart.py new file mode 100644 index 000000000..178176eaa --- /dev/null +++ b/tests/charts/test_saw_kubernetes_proxy_chart.py @@ -0,0 +1,67 @@ +import json +from pathlib import Path +import shutil +import pytest +import yaml +from helm_cache import run_helm + +ROOT = Path(__file__).resolve().parents[2] +HELM = shutil.which('helm') +pytestmark = [pytest.mark.ci_external, pytest.mark.skipif(not HELM, reason='Helm required')] + + +def render(*args, enabled=True): + return run_helm([HELM, 'template', 'proxy', str(ROOT / 'charts/saw-kubernetes-proxy'), + '-n', 'saw-test-trusted', '--set', 'enabled=' + str(enabled).lower(), + '--set', 'gateway=saw-test,targetNamespace=saw-test,ownerID=67a9891d-e281-442f-86c3-41771f5ce049', + '--set', 'policyConfigMap=approved-policy,credentialSecretNames[0]=inference', + '--set-string', 'policyDigest=' + 'a'*64, + '--set-json', 'network.bootstrapEgress=[{"to":[{"ipBlock":{"cidr":"10.0.0.1/32"}}],"ports":[{"port":443}]}]', + '--set-json', 'network.proxyEgress=[{"to":[{"ipBlock":{"cidr":"0.0.0.0/0"}}],"ports":[{"port":443}]}]', + *args], capture_output=True, text=True) + + +def test_isolation_and_retained_public_projection(): + result = render() + assert result.returncode == 0, result.stderr + docs = list(yaml.safe_load_all(result.stdout)) + assert not any(d['kind'] == 'Secret' for d in docs) + deployment = next(d for d in docs if d['kind'] == 'Deployment') + pod = deployment['spec']['template']['spec'] + assert pod['automountServiceAccountToken'] is False + assert not any(v.get('projected') for v in pod['volumes']) + assert pod['containers'][0]['args'] == ['serve'] + job = next(d for d in docs if d['kind'] == 'Job')['spec']['template']['spec'] + assert job['initContainers'][0]['args'] == ['init-ca'] + assert job['automountServiceAccountToken'] is False + for c in [*job['initContainers'], *job['containers'], *pod['containers']]: + assert '@sha256:' in c['image'] + assert c['securityContext'] == dict(allowPrivilegeEscalation=False, + readOnlyRootFilesystem=True, capabilities={'drop': ['ALL']}) + runtime = next(d for d in docs if d['kind'] == 'Role' and d['metadata']['namespace']=='saw-test') + assert all(r['resources'] == ['configmaps'] and set(r['verbs']) <= {'get','create'} for r in runtime['rules']) + trusted = next(d for d in docs if d['kind']=='Role' and d['metadata']['namespace']=='saw-test-trusted') + assert not any('update' in r['verbs'] for r in trusted['rules'] if 'secrets' in r['resources']) + assert not any(d['kind']=='RoleBinding' and d['subjects'][0]['name']=='saw-test-credential-proxy' for d in docs) + cm = next(d for d in docs if d['kind']=='ConfigMap') + assert json.loads(cm['data']['config.json'])['name']=='saw-test-proxy-ca' + assert cm['data']['project_ca.py'].rstrip()==(ROOT/'charts/saw-kubernetes-proxy/files/project_ca.py').read_text().rstrip() + np = next(d for d in docs if d['kind']=='NetworkPolicy' and d['metadata']['name']=='saw-test-credential-proxy') + peers = np['spec']['ingress'][0]['from'] + assert all(set(peer)=={'namespaceSelector','podSelector'} for peer in peers) + assert peers[1]['podSelector']['matchLabels']=={'openshell.ai/gateway-id':'saw-test','openshell.ai/boundary-role':'supervisor'} + service = next(d for d in docs if d['kind']=='Service') + assert [p['port'] for p in service['spec']['ports']]==[3128] + + +def test_opt_in(): + result=render(enabled=False) + assert result.returncode==0 and not result.stdout.strip() + + +@pytest.mark.parametrize('value', ['targetNamespace=saw-test-trusted','gateway=kube-system', + 'ownerID=00000000-0000-0000-0000-000000000000','image=repo:latest','helperImage=repo:latest', + 'policyConfigMap=','policyDigest=','credentialSecretNames[1]=inference']) +def test_fail_closed(value): + result=render('--set-string',value) + assert result.returncode != 0 and not result.stdout.strip() diff --git a/tests/charts/test_saw_runtime_policies.py b/tests/charts/test_saw_runtime_policies.py new file mode 100644 index 000000000..8b0bfa237 --- /dev/null +++ b/tests/charts/test_saw_runtime_policies.py @@ -0,0 +1,259 @@ +"""Offline guardrails for native runtime policies; no cluster/CEL/CNI claim. + +Helm rejects unsafe/missing inputs. The packet cases exercise the rendered +ordered ANP rules, including ports and peer scope, against a small reference +interpreter. Kubernetes CEL type checking and live OVN enforcement remain +required integration tests described in the chart README. +""" + +import ipaddress +import shutil +from pathlib import Path + +import pytest +import yaml +from helm_cache import run_helm + +ROOT = Path(__file__).resolve().parents[2] +CHART = ROOT / "charts/saw-runtime-policies" +HELM = shutil.which("helm") +APIS = ( + "policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy", + "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy", +) +pytestmark = [pytest.mark.ci_external, pytest.mark.skipif(not HELM, reason="helm is not installed")] + + +def helm_template(*args, enabled=True, apis=APIS): + command = [HELM, "template", "runtime-test", str(CHART)] + if enabled: + command += ["-f", str(CHART / "tests/container-values.yaml")] + for api in apis: + command += ["--api-versions", api] + return run_helm(command + list(args), capture_output=True, text=True) + + +def render(*args, **kwargs): + result = helm_template(*args, **kwargs) + assert result.returncode == 0, result.stderr + return {doc["kind"]: doc for doc in yaml.safe_load_all(result.stdout) if doc} + + +def rejected(tmp_path, values, message=None): + path = tmp_path / "invalid.yaml" + path.write_text(yaml.safe_dump(values)) + result = helm_template("-f", str(path)) + assert result.returncode != 0, values + if message: + assert message in result.stderr, result.stderr + + +def test_disabled_chart_requires_no_apis_and_creates_nothing(): + assert render(enabled=False, apis=()) == {} + + +@pytest.mark.parametrize("api", APIS) +def test_missing_native_api_fails_before_render(api): + result = helm_template(apis=tuple(candidate for candidate in APIS if candidate != api)) + assert result.returncode != 0 + assert "API is required" in result.stderr + + +def test_only_native_policies_with_exact_namespace_and_supervisor_scope(): + docs = render() + assert set(docs) == {"AdminNetworkPolicy", "ValidatingAdmissionPolicy", "ValidatingAdmissionPolicyBinding"} + anp, vap, binding = (docs[kind] for kind in ( + "AdminNetworkPolicy", "ValidatingAdmissionPolicy", "ValidatingAdmissionPolicyBinding")) + ns = {"matchLabels": {"kubernetes.io/metadata.name": "saw-alice"}} + assert anp["spec"]["subject"] == {"pods": { + "namespaceSelector": ns, + "podSelector": {"matchLabels": { + "openshell.ai/boundary-role": "supervisor", + "openshell.ai/gateway-id": "saw-alice-gateway"}}, + }} + assert "ingress" not in anp["spec"] + assert anp["spec"]["priority"] == 70 + assert vap["spec"]["failurePolicy"] == "Fail" + assert vap["spec"]["matchConstraints"]["namespaceSelector"] == ns + assert vap["spec"]["matchConstraints"]["resourceRules"] == [{ + "apiGroups": [""], "apiVersions": ["v1"], "operations": ["CREATE", "UPDATE"], "resources": ["pods"]}] + assert binding["spec"]["matchResources"]["namespaceSelector"] == ns + assert binding["spec"]["policyName"] == vap["metadata"]["name"] + assert binding["spec"]["validationActions"] == ["Deny"] + for kind, doc in docs.items(): + assert "namespace" not in doc["metadata"] + assert doc["metadata"]["annotations"]["argocd.argoproj.io/sync-options"] == "Prune=false" + assert int(doc["metadata"]["annotations"]["argocd.argoproj.io/sync-wave"]) < 0 + assert not any(key in doc["spec"] for key in ["mutate", "mutations", "generate"]), kind + + +def test_approved_traffic_passes_to_stock_network_policy_and_everything_else_denied(): + rules = render()["AdminNetworkPolicy"]["spec"]["egress"] + assert all(rule["action"] == "Pass" and rule["ports"] for rule in rules[:-1]) + assert rules[-1] == {"name": "deny-other-egress", "action": "Deny", + "to": [{"networks": ["0.0.0.0/0", "::/0"]}]} + assert all(rule["action"] != "Allow" for rule in rules) + + +def label_match(selector, labels): + # Fixtures intentionally use exact matchLabels. This is a test interpreter, + # not an implementation of the full Kubernetes LabelSelector language. + assert set(selector) == {"matchLabels"} + return all(labels.get(key) == value for key, value in selector["matchLabels"].items()) + + +def peer_match(peer, ip, namespace, labels): + if "networks" in peer: + return any(ipaddress.ip_address(ip) in ipaddress.ip_network(cidr) for cidr in peer["networks"]) + ns = {"kubernetes.io/metadata.name": namespace} + if "namespaces" in peer: + return label_match(peer["namespaces"], ns) + return (label_match(peer["pods"]["namespaceSelector"], ns) + and label_match(peer["pods"]["podSelector"], labels)) + + +def packet_action(anp, *, ip, namespace="outside", labels=None, port=443, protocol="TCP"): + for rule in anp["spec"]["egress"]: + if "ports" in rule and not any( + entry["portNumber"] == {"port": port, "protocol": protocol} for entry in rule["ports"] + ): + continue + if any(peer_match(peer, ip, namespace, labels or {}) for peer in rule["to"]): + return rule["action"] + return None + + +@pytest.mark.parametrize("packet,expected", [ + ({"ip": "192.0.2.10"}, "Pass"), + ({"ip": "2001:db8::10"}, "Pass"), + ({"ip": "192.0.2.10", "port": 22}, "Deny"), + ({"ip": "192.0.2.10", "protocol": "UDP"}, "Deny"), + ({"ip": "192.0.2.11"}, "Deny"), + ({"ip": "2001:db8::11"}, "Deny"), + ({"ip": "10.2.0.3", "namespace": "saw-alice", "labels": { + "openshell.ai/boundary-role": "workload"}, "port": 5500}, "Pass"), + ({"ip": "10.2.0.3", "namespace": "saw-bob", "labels": { + "openshell.ai/boundary-role": "workload"}, "port": 5500}, "Deny"), + ({"ip": "10.2.0.3", "namespace": "saw-alice", "labels": { + "openshell.ai/boundary-role": "supervisor"}, "port": 5500}, "Deny"), + ({"ip": "10.2.0.3", "namespace": "saw-alice", "labels": { + "openshell.ai/boundary-role": "workload"}, "port": 22}, "Deny"), + ({"ip": "10.2.0.4", "namespace": "openshift-dns", "labels": { + "dns.operator.openshift.io/daemonset-dns": "default"}, "port": 5353, "protocol": "UDP"}, "Pass"), + ({"ip": "10.2.0.4", "namespace": "openshift-dns", "labels": {}, + "port": 5353, "protocol": "UDP"}, "Deny"), +]) +def test_rendered_anp_destination_and_port_boundaries(packet, expected): + assert packet_action(render()["AdminNetworkPolicy"], **packet) == expected + + +def expressions(docs): + return [" ".join(entry["expression"].split()) + for entry in docs["ValidatingAdmissionPolicy"]["spec"]["validations"]] + + +def test_container_mode_requires_default_runtime_for_both_roles(): + checks = expressions(render()) + for role in ["workload", "supervisor"]: + assert (f"variables.role != '{role}' || !has(object.spec.runtimeClassName) || " + "object.spec.runtimeClassName == ''") in checks + + +def test_supervisor_accepts_stock_gateway_names_only(): + validations = render()["ValidatingAdmissionPolicy"]["spec"]["validations"] + rule, = [item["expression"] for item in validations + if item["message"] == "Supervisor executable, TLS projection and token must match the stock contract."] + assert "'OPENSHELL_GATEWAY_TLS_SERVER_NAME'" in rule + assert "'OPENSHELL_GATEWAY_NAME'" in rule + assert "object.spec.containers[0].env.all(e, e.name in [" in rule + assert "AWS_SECRET_ACCESS_KEY" not in rule + + +def test_identity_fences_cover_creation_and_label_removal_on_update(): + vap = render()["ValidatingAdmissionPolicy"]["spec"] + variables = {item["name"]: item["expression"] for item in vap["variables"]} + assert "oldObject != null" in variables["previousRole"] + assert "oldObject != null" in variables["previousSandboxAccount"] + assert "variables.previousSandboxAccount" in variables["runtimePod"] + assert "variables.previousRole in ['workload', 'supervisor']" in variables["runtimePod"] + checks = expressions(render()) + assert any("variables.previousRole == variables.role" in check for check in checks) + assert any('request.userInfo.username == "system:serviceaccount:saw-alice:saw-alice-gateway"' + in check for check in checks) + assert any("r.kind == 'Sandbox'" in check for check in checks) + assert all("r.controller" not in check for check in checks if "r.kind == 'Sandbox'" in check) # stock supervisor owner is non-controlling + assert any("variables.sandboxAccount && variables.role in ['workload', 'supervisor']" in check for check in checks) + + +@pytest.mark.parametrize("namespace", ["", "default", "openshift-dns", "openshift", "kube-system", "kube-public", "kube-node-lease", "kube-custom", "ALL", "a" * 51]) +def test_empty_global_or_system_namespace_is_rejected(tmp_path, namespace): + rejected(tmp_path, {"targetNamespace": namespace}, "targetNamespace") + + +@pytest.mark.parametrize("values", [ + {"gatewayId": ""}, {"gatewayServiceAccount": ""}, {"sandboxServiceAccount": ""}, + {"network": {"priority": None}}, {"network": {"priority": -1}}, + {"network": {"priority": 100}}, {"network": {"priority": 1.5}}, + {"network": {"egress": []}}, {"mode": "confidential"}, + {"mode": "kata"}, {"mode": "kata", "kata": {"workloadRuntimeClassName": "bad/class"}}, + {"kata": {"workloadRuntimeClassName": "kata"}}, + {"kata": {"supervisorRuntimeClassName": "kata"}}, + {"kata": {"kyvernoMutationEnabled": True}}, +]) +def test_missing_or_unsupported_inputs_fail_closed(tmp_path, values): + rejected(tmp_path, values) + + +@pytest.mark.parametrize("rule", [ + {"name": "bad", "to": [{}], "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}]}, + {"name": "bad", "to": [{"namespaces": {}}], "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}]}, + {"name": "bad", "to": [{"pods": {"namespaceSelector": {}, "podSelector": {}}}], + "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}]}, + {"name": "bad", "to": [{"networks": ["0.0.0.0/0"]}], "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}]}, + {"name": "bad", "to": [{"networks": ["::/0"]}], "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}]}, + {"name": "bad", "to": [{"networks": ["192.0.2.0/24"]}], "ports": []}, + {"name": "bad", "to": [{"networks": ["192.0.2.0/24"]}], "ports": [{"portNumber": {"protocol": "TCP", "port": 0}}]}, + {"name": "bad", "to": [{"networks": ["192.0.2.0/24"]}], "ports": [{"portNumber": {"protocol": "TCP", "port": 65536}}]}, + {"name": "bad", "to": [{"networks": ["192.0.2.0/24"]}], "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}], "action": "Allow"}, + {"name": "deny-other-egress", "to": [{"networks": ["192.0.2.0/24"]}], "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}]}, +]) +def test_policy_cannot_be_widened_with_empty_selectors_or_caller_actions(tmp_path, rule): + rejected(tmp_path, {"network": {"egress": [rule]}}) + + +def test_duplicate_rule_names_are_rejected(tmp_path): + rule = {"name": "same", "to": [{"networks": ["192.0.2.0/24"]}], + "ports": [{"portNumber": {"protocol": "TCP", "port": 443}}]} + rejected(tmp_path, {"network": {"egress": [rule, rule]}}, "must be unique") + + +def test_priority_zero_is_a_valid_explicit_allocation(): + assert render("--set", "network.priority=0")["AdminNetworkPolicy"]["spec"]["priority"] == 0 + + +def test_gateway_proxy_identity_requires_pinned_statefulset_and_controller(): + checks = expressions(render()) + assert any("r.kind == 'StatefulSet'" in c and "r.controller" in c and + 'r.name == "saw-alice-gateway-gateway"' in c for c in checks) + assert any("variables.previousGatewayClaim == variables.gatewayClaim" in c for c in checks) + assert any("system:serviceaccount:kube-system:statefulset-controller" in c and + "request.operation != 'CREATE'" in c for c in checks) + + +def test_kata_enforces_reviewed_runtime_at_sandbox_and_pod(): + result = helm_template("--set", "mode=kata", "--set", "workloadRuntimeClassName=saw-kata") + assert result.returncode == 0, result.stderr + policies = [d["spec"] for d in yaml.safe_load_all(result.stdout) + if d and d["kind"] == "ValidatingAdmissionPolicy"] + assert len(policies) == 2 + for policy in policies: + assert policy["failurePolicy"] == "Fail" + checks = [entry["expression"] for entry in policy["validations"]] + resource = policy["matchConstraints"]["resourceRules"][0]["resources"][0] + if resource == "sandboxes": + assert any('podTemplate.spec.runtimeClassName == "saw-kata"' in check for check in checks) + else: + assert resource == "pods" + assert any('object.spec.runtimeClassName == "saw-kata"' in check for check in checks) + assert any("variables.role != 'supervisor' || !has(object.spec.runtimeClassName)" in check + for check in checks) diff --git a/tests/charts/test_saw_users_chart.py b/tests/charts/test_saw_users_chart.py index b609ec70b..32d18e428 100644 --- a/tests/charts/test_saw_users_chart.py +++ b/tests/charts/test_saw_users_chart.py @@ -69,6 +69,39 @@ def helm_values(application): return yaml.safe_load(application["spec"]["source"]["helm"]["values"]) +def test_container_policy_is_derived_from_documented_hosts(tmp_path): + site = yaml.safe_load((ROOT / "tests/fixtures/container-site.yaml").read_text())["portal"]["containerSites"]["alice"] + del site["providerPolicy"] + user = { + "name": "alice", "registration": "a" * 16, + "installationID": "22222222-2222-4222-8222-222222222222", + "ownerSubject": site["subject"], "profiles": ["data-science"], + "intentGeneration": 1, "vaultPrefix": "secret/data/hub/saw-alice/" + "a" * 16, + "pruneOnRemove": True, "values": {"deployment": {"mode": "container"}}, + } + docs = docs_from(render_file(tmp_path, [user], {"containerSites": {"alice": site}})) + rendered_site = helm_values(app(docs, "saw-alice"))["site"] + assert rendered_site["providerPolicy"]["allowHosts"] == site["providerAllowHosts"] + + +def test_static_owner_revision_pin_selects_all_children_and_preserves_sibling(tmp_path): + docs = docs_from(render_file(tmp_path, [ALICE, {"name": "bob"}], + {"workspaceRevisions": {"alice": "a"*40}})) + for application in by_kind(docs, "Application"): + owner = application["metadata"]["labels"]["openshell.pattern/owner"] + source = application["spec"]["source"] + assert source["targetRevision"] == ("a"*40 if owner == "alice" else "main") + assert source["repoURL"] == "https://example.com/secure-agent-workspace.git" + + +@pytest.mark.parametrize("pins", [[], {"alice": "branch"}, {"../alice": "a"*40}, + {"alice": {"repoURL": "https://other.example/repo"}}]) +def test_static_revision_pins_fail_before_rendering_partial_workloads(tmp_path, pins): + result = render_file(tmp_path, [ALICE], {"workspaceRevisions": pins}) + assert result.returncode != 0 + assert "workspaceRevisions requires" in result.stderr + + @pytest.mark.ci_external @pytest.mark.parametrize("override", [None, "quay.io/example/proxy@sha256:" + "1" * 64]) def test_user_inherits_published_proxy_image_or_default_override(tmp_path, override): @@ -721,3 +754,54 @@ def test_direct_rendered_values_validate_real_provider_keys(tmp_path): assert result.returncode == 0, result.stdout + result.stderr assert "mode: direct" in result.stdout assert "direct-sentinel-key" not in result.stdout + result.stderr + + +# -- deployment backend contract ---------------------------------------------- + +@pytest.mark.ci_external +def test_mode_omission_preserves_child_values(tmp_path): + docs = docs_from(render_file(tmp_path, [ALICE])) + assert "deployment" not in helm_values(app(docs, "saw-alice")) + + +@pytest.mark.ci_external +def test_user_mode_overrides_default_and_reaches_child_chart(tmp_path): + docs = docs_from(render_file(tmp_path, + [{"name": "alice", "values": {"deployment": {"mode": "vm"}}}], + {"defaults": {"openshellSaw": {"deployment": {"mode": "container"}}}})) + assert helm_values(app(docs, "saw-alice"))["deployment"] == {"mode": "vm"} + + +@pytest.mark.ci_external +@pytest.mark.parametrize("mode", ["container", "kata", "confidential", "typo", ""]) +@pytest.mark.parametrize("level", ["default", "user"]) +def test_unsupported_effective_mode_rejected_before_child_apps(tmp_path, mode, level): + users = [{"name": "alice"}] + extra = {} + if level == "default": + extra = {"defaults": {"openshellSaw": {"deployment": {"mode": mode}}}} + else: + users[0]["values"] = {"deployment": {"mode": mode}} + result = render_file(tmp_path, users, extra) + assert result.returncode != 0 + assert "deployment.mode" in result.stderr + assert not result.stdout.strip() + + +@pytest.mark.ci_external +@pytest.mark.parametrize("level", ["default", "user"]) +@pytest.mark.parametrize("deployment", [ + {"mode": False}, {"mode": 1}, {"mode": None}, {"mode": []}, + {"mod": "container"}, "container", [], None, +]) +def test_invalid_deployment_shape_rejected(tmp_path, deployment, level): + users = [{"name": "alice"}] + extra = {} + if level == "default": + extra = {"defaults": {"openshellSaw": {"deployment": deployment}}} + else: + users[0]["values"] = {"deployment": deployment} + result = render_file(tmp_path, users, extra) + assert result.returncode != 0 + assert "deployment" in result.stderr + assert not result.stdout.strip() diff --git a/tests/charts/test_workspace_finalizers.py b/tests/charts/test_workspace_finalizers.py new file mode 100644 index 000000000..0cc396013 --- /dev/null +++ b/tests/charts/test_workspace_finalizers.py @@ -0,0 +1,49 @@ +"""Both entry points preserve Argo hook state without hiding desired-state drift.""" +import json +import shutil +import subprocess + +import pytest +from test_portal_container import GLOBAL, SITE, U, render, user + + +@pytest.mark.parametrize('entry', ['portal', 'static']) +def test_hook_state_does_not_fight_parent_sync_or_hide_cleanup_drift(entry): + if entry == 'portal': + docs = render('openshell-rhdh', 'portal', 'rhdh', {}) + else: + docs = render('saw-users', 'workspaces', 'saw-installation', { + 'global': GLOBAL, 'users': [user()], 'containerSites': {U: SITE}, + 'projection': {'enabled': True, 'applicationSetControllerLabels': { + 'app.kubernetes.io/name': 'vp-gitops-applicationset-controller'}, + 'egress': [{'ports': [{'protocol': 'TCP', 'port': 443}]}]}}) + appset, = [d for d in docs if d['kind'] == 'ApplicationSet'] + spec = appset['spec']['template']['spec'] + rule, = spec['ignoreDifferences'] + assert set(rule) == {'group', 'kind', 'jqPathExpressions'} + assert (rule['group'], rule['kind']) == ('argoproj.io', 'Application') + assert 'RespectIgnoreDifferences=true' in spec['syncPolicy']['syncOptions'] + expression, = rule['jqPathExpressions'] + assert expression == ('.metadata.finalizers[]? | select(. == ' + '"pre-delete-finalizer.argocd.argoproj.io" or . == ' + '"pre-delete-finalizer.argocd.argoproj.io/cleanup")') + if not shutil.which('jq'): + pytest.skip('jq required for semantic diff-normalization check') + + def normalize(value): + p = subprocess.run(['jq', '-c', 'del(' + expression + ')'], + input=json.dumps(value), text=True, capture_output=True, check=True) + return json.loads(p.stdout) + + desired = {'metadata': {'finalizers': ['resources-finalizer.argocd.argoproj.io/foreground']}, + 'spec': {'source': {'targetRevision': 'reviewed'}}} + live = json.loads(json.dumps(desired)) + live['metadata']['finalizers'] += ['pre-delete-finalizer.argocd.argoproj.io', + 'pre-delete-finalizer.argocd.argoproj.io/cleanup'] + assert normalize(live) == desired + live['metadata']['finalizers'].pop(0) + assert normalize(live) != desired # Removing cascade cleanup must still be repaired. + live = json.loads(json.dumps(desired)) + live['spec']['source']['targetRevision'] = 'unreviewed' + assert normalize(live) != desired + assert normalize({'metadata': {}}) == {'metadata': {}} diff --git a/tests/charts/test_workspace_relay.py b/tests/charts/test_workspace_relay.py new file mode 100644 index 000000000..71e8fadb4 --- /dev/null +++ b/tests/charts/test_workspace_relay.py @@ -0,0 +1,154 @@ +import shutil +import socket +import subprocess +import sys +import time +from pathlib import Path + +import pytest +import yaml +from helm_cache import run_helm + +CHART = Path(__file__).resolve().parents[2] / "charts/saw-workspace-relay" +pytestmark = [pytest.mark.ci_external, pytest.mark.skipif(not shutil.which("helm"), reason="Helm required")] + + +def test_secret_and_membership_boundaries(tmp_path): + values = dict(enabled=True, clientID="site-agent", keycloakNamespace="kc", keycloakProvisionerSecret="kc-admin", + issuer="https://kc.example/realms/openshell", trustedNamespace="trusted", gatewayNamespace="runtime", + installationID="installation", gateway="gateway", workspace="agent", workspaceID="old-id", + sandbox="assistant", ownerLogin="alice", ownerSubject="owner", subject="relay", + imageBuildNamespace="build", sdkImage="example/sdk@sha256:"+"f"*64, + host="alice.apps.example", egress=[{"ports":[{"port":443}]}]) + path = tmp_path / "values.yaml" + path.write_text(yaml.safe_dump(values)) + output = run_helm(["helm","template","relay",str(CHART),"-n","ui","-f",str(path)], + capture_output=True,text=True) + assert output.returncode == 0, output.stderr + resources = list(yaml.safe_load_all(output.stdout)) + deployment = next(d for d in resources if d["kind"] == "Deployment") + pod = deployment["spec"]["template"]["spec"] + assert pod["automountServiceAccountToken"] is False + assert {v["secret"]["secretName"] for v in pod["volumes"] if "secret" in v} == {"relay-credentials"} + containers = {c["name"]:c for c in pod["containers"]} + assert "credential" not in {v["name"] for v in containers["forward"]["volumeMounts"]} + assert "--local 127.0.0.1:28789" in containers["forward"]["args"][0] + assert containers["transport"]["command"] == ["python3", "-I", "/config/saw_ui_limit.py", "18789", "28789", "16", "--connect-only"] + assert {m["name"] for m in containers["transport"]["volumeMounts"]} == {"config", "proof"} + assert "--oidc-email-claim=sub" in containers["browser-login"]["args"] + assert containers["transport"]["securityContext"] == containers["forward"]["securityContext"] + canonical = CHART.parent / "openshell-saw/files/installer/saw_ui_limit.py" + assert (CHART / "files/saw_ui_limit.py").read_bytes() == canonical.read_bytes() + config = next(d for d in resources if d["kind"] == "ConfigMap" and d["metadata"]["name"] == "relay") + assert config["data"]["saw_ui_limit.py"] == canonical.read_text() + assert "--authenticated-emails-file=/config/users" in containers["browser-login"]["args"] + jobs = {d["metadata"]["namespace"]:d["spec"]["template"]["spec"] for d in resources if d["kind"]=="Job"} + assert jobs["trusted"]["automountServiceAccountToken"] is False + assert not any("secret" in v for v in jobs["trusted"]["volumes"]) + volumes = {v["name"]: v for v in jobs["trusted"]["volumes"]} + assert volumes["identity"]["configMap"]["name"] == "gateway-gateway-ca" + assert volumes["admin-session"]["emptyDir"] == {"medium": "Memory", "sizeLimit": "4Mi"} + runtime_role = next(d for d in resources if d["kind"] == "Role" and d["metadata"]["name"] == "relay-admin-session") + assert runtime_role["rules"] == [{"apiGroups": [""], "resources": ["secrets"], + "resourceNames": ["gateway-client-ca"], "verbs": ["get", "update"]}] + binding = next(d for d in resources if d["kind"] == "RoleBinding" and d["metadata"]["name"] == "relay-admin-session") + assert binding["subjects"] == [{"kind": "ServiceAccount", "name": "relay", "namespace": "trusted"}] + assert [v["secret"]["secretName"] for v in jobs["kc"]["volumes"] if "secret" in v] == ["kc-admin"] + role = next(d for d in resources if d["kind"]=="Role" and d["metadata"].get("namespace", "ui") == "ui") + assert role["rules"] == [{"apiGroups":[""],"resources":["secrets"],"resourceNames":["relay-credentials"],"verbs":["get","patch"]}] + + +def test_rendered_transport_starts_beside_projected_token_module(tmp_path): + values = dict(enabled=True, clientID="site-agent", keycloakNamespace="kc", keycloakProvisionerSecret="kc-admin", + issuer="https://kc.example/realms/openshell", trustedNamespace="trusted", gatewayNamespace="runtime", + installationID="installation", gateway="gateway", workspace="agent", workspaceID="workspace-id", + sandbox="assistant", ownerLogin="alice", ownerSubject="owner", subject="relay", + imageBuildNamespace="build", sdkImage="example/sdk@sha256:"+"f"*64, + host="alice.apps.example", egress=[{"ports":[{"port":443}]}]) + rendered = subprocess.run(["helm", "template", "relay", str(CHART), "-n", "ui", "-f", "-"], + input=yaml.safe_dump(values), capture_output=True, text=True, check=True) + docs = list(yaml.safe_load_all(rendered.stdout)) + config = next(d for d in docs if d["kind"] == "ConfigMap" and d["metadata"]["name"] == "relay") + pod = next(d for d in docs if d["kind"] == "Deployment")["spec"]["template"]["spec"] + command = list(next(c for c in pod["containers"] if c["name"] == "transport")["command"]) + script = tmp_path / "saw_ui_limit.py" + script.write_text(config["data"]["saw_ui_limit.py"]) + # Projected token.py is an OIDC helper, not Python's stdlib token module. + # asyncio -> logging -> tokenize must not import it from the script directory. + (tmp_path / "token.py").write_text("raise RuntimeError('projected-token-shadow')\n") + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + command[0] = sys.executable + command[command.index("/config/saw_ui_limit.py")] = str(script) + command[command.index("18789")] = str(port) + with subprocess.Popen(command, cwd=tmp_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as process: + try: + deadline = time.monotonic() + 5 + while True: + if process.poll() is not None: + _, error = process.communicate() + pytest.fail("rendered transport exited before listening: " + error) + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + assert time.monotonic() < deadline, "transport did not start listening" + time.sleep(0.02) + finally: + process.terminate() + process.communicate(timeout=3) + # The regression fixture must reproduce the original failure without -I. + unsafe = subprocess.run([arg for arg in command if arg != "-I"], cwd=tmp_path, + capture_output=True, text=True, timeout=5) + assert unsafe.returncode != 0 and "projected-token-shadow" in unsafe.stderr + + +def test_relay_jobs_change_with_images_and_mount_their_exact_scripts(): + values = dict(enabled=True, clientID="site-agent", keycloakNamespace="kc", keycloakProvisionerSecret="kc-admin", + issuer="https://kc.example/realms/openshell", trustedNamespace="trusted", gatewayNamespace="runtime", + installationID="installation", gateway="gateway", workspace="agent", workspaceID="workspace-id", + sandbox="assistant", ownerLogin="alice", ownerSubject="owner", subject="relay", + imageBuildNamespace="build", sdkImage="example/sdk@sha256:"+"f"*64, + host="alice.apps.example", egress=[{"ports":[{"port":443}]}]) + def jobs(v): + result = subprocess.run(["helm", "template", "relay", str(CHART), "-n", "ui", "-f", "-"], + input=yaml.safe_dump(v), capture_output=True, text=True, check=True) + docs = list(yaml.safe_load_all(result.stdout)) + configs = {(o["metadata"].get("namespace", "ui"), o["metadata"]["name"]): o + for o in docs if o["kind"] == "ConfigMap"} + names = set() + for job in [o for o in docs if o["kind"] == "Job"]: + script, = [v for v in job["spec"]["template"]["spec"]["volumes"] if v["name"] == "script"] + assert configs[(job["metadata"]["namespace"], script["configMap"]["name"])]["immutable"] is True + names.add(job["metadata"]["name"]) + return names + initial = jobs(values) + assert jobs(values) == initial + changed = jobs({**values, "sdkImage": "example/sdk@sha256:"+"a"*64, + "pythonImage": "example/python@sha256:"+"a"*64}) + assert initial.isdisjoint(changed) + + +def test_relay_replaces_spoofed_headers_and_requires_authenticated_subject(tmp_path): + import asyncio + import importlib.util + spec = importlib.util.spec_from_file_location("review_transport", CHART/"files"/"saw_ui_limit.py") + transport = importlib.util.module_from_spec(spec) + spec.loader.exec_module(transport) + proof = tmp_path/"proof" + proof.write_bytes(b"a"*64) + async def request(headers): + reader = asyncio.StreamReader() + reader.feed_data(b"GET /ws HTTP/1.1\r\n"+headers+b"\r\n\r\n") + reader.feed_eof() + return await transport.relay_request(reader, proof, "owner-sub") + raw = asyncio.run(request(b"X-Forwarded-Email: owner-sub\r\nX-Saw-Relay-Identity: forged")) + assert b"Connection: close\r\n" in raw + upgraded = asyncio.run(request(b"X-Forwarded-Email: owner-sub\r\nUpgrade: websocket")) + assert b"Connection: Upgrade\r\n" in upgraded + assert raw.count(b"X-Saw-Relay-Identity: ") == 1 + assert b"forged" not in raw and b"owner-sub."+b"a"*64 in raw + for headers in [b"X-Forwarded-Email: reused-login", b"X-Forwarded-Email: owner-sub\r\nX-Forwarded-Email: foreign"]: + with pytest.raises(ValueError): + asyncio.run(request(headers)) diff --git a/tests/fixtures/audited-profile.json b/tests/fixtures/audited-profile.json new file mode 100644 index 000000000..1c612d8f3 --- /dev/null +++ b/tests/fixtures/audited-profile.json @@ -0,0 +1,36 @@ +{ + "description": "OpenClaw chat with native tool approval and retained audit history", + "harnessRequired": false, + "secrets": { + "inference": { + "fields": [ + { + "key": "api_key", + "kind": "secret", + "required": true, + "title": "API key" + } + ], + "provider": "openai", + "providers": [ + "openai" + ] + } + }, + "workspaces": [ + { + "description": "OpenClaw chat with native tool approval and retained audit history", + "enabled": true, + "name": "agent", + "sandboxes": [ + { + "enabled": true, + "harnessRequired": false, + "name": "audited", + "type": "openclaw", + "uiRoute": true + } + ] + } + ] +} diff --git a/tests/fixtures/briefing-apis.mjs b/tests/fixtures/briefing-apis.mjs new file mode 100644 index 000000000..b45f35ebd --- /dev/null +++ b/tests/fixtures/briefing-apis.mjs @@ -0,0 +1,127 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import http from "node:http"; +import https from "node:https"; +const day = Math.floor(new Date(new Date().setHours(0, 0, 0, 0)).getTime() / 1000); +const gmailCount = Number(process.env.FAKE_GMAIL_COUNT || 1); +const slackCount = Number(process.env.FAKE_SLACK_COUNT || 1); +const channelPages = Number(process.env.FAKE_CHANNEL_PAGES || 1); +const slackThreads = () => process.env.FAKE_SLACK_THREADS + ? JSON.parse(readFileSync(process.env.FAKE_SLACK_THREADS, "utf8")) : []; +const failSpec = process.env.FAKE_FAIL_ONCE || ""; +const failed = new Set(); +const requests = []; +const reply = (res, body, status = 200) => { + const send = () => { + res.writeHead(status, {"content-type": "application/json"}); + res.end(JSON.stringify(body)); + }; + const delay = Number(process.env.FAKE_DELAY_MS || 0); + if (delay > 0) setTimeout(send, delay); + else send(); +}; +const record = (url) => { + requests.push({path: url.pathname, query: Object.fromEntries(url.searchParams)}); + if (process.env.FAKE_REQUEST_LOG) writeFileSync(process.env.FAKE_REQUEST_LOG, JSON.stringify(requests)); +}; +const pageNumber = (cursor) => cursor ? Number(cursor) : 0; +const nextCursor = (page, end, count) => { + if (process.env.FAKE_REPEAT_CURSOR) return "repeat"; + return end < count ? String(page + 1) : ""; +}; +const failOnce = (path, page, res) => { + const key = `${path}:${page}`; + if (failSpec === key && !failed.has(key)) { + failed.add(key); + reply(res, {ok: false, error: "server_error"}, 500); + return true; + } + return false; +}; +const handler = (req, res) => { + const url = new URL(req.url, "http://x"); + record(url); + const expected = url.pathname.startsWith("/slack") ? "slack-ph" : "gmail-ph"; + if (req.headers.authorization !== `Bearer ${expected}`) + return reply(res, {ok: false, error: "invalid_auth"}, 401); + if (url.pathname === "/slack/conversations.list") { + const all = channelPages > 1 + ? [{id: "C0", name: "random", is_member: false}, {id: "C1", name: "eng", is_member: true}] + : [{id: "C1", name: "eng", is_member: true}, {id: "C2", name: "random", is_member: false}]; + const page = pageNumber(url.searchParams.get("cursor")); + const limit = Number(url.searchParams.get("limit") || all.length); + const start = page * limit, end = Math.min(start + limit, all.length); + if (failOnce(url.pathname, page + 1, res)) return; + const next = nextCursor(page, end, all.length); + const body = {ok: true, channels: all.slice(start, end)}; + if (next) { body.has_more = true; body.response_metadata = {next_cursor: next}; } + else if (process.env.FAKE_HAS_MORE_NO_CURSOR) body.has_more = true; + return reply(res, body); + } + if (url.pathname === "/slack/conversations.history") { + const oldest = Number(url.searchParams.get("oldest")); + const generated = Array.from({length: slackCount}, (_, i) => ({ + ts: String(day + 60 + i), user: "U1", + text: i === 0 ? "Can you review the PR today?" : `Slack message ${i + 1}`, + })); + const threaded = slackThreads().flatMap((thread) => [ + {...thread, thread_ts: thread.ts, reply_count: thread.replies.length, + latest_reply: thread.replies.reduce((latest, m) => Number(m.ts) > Number(latest) ? m.ts : latest, "0")}, + ...thread.replies.filter((m) => m.subtype === "thread_broadcast") + .map((m) => ({...m, thread_ts: thread.ts})), + ]); + const all = [...generated, ...threaded].sort((a, b) => Number(b.ts) - Number(a.ts)); + const page = pageNumber(url.searchParams.get("cursor")); + const limit = Number(url.searchParams.get("limit") || all.length); + const filtered = all.filter((m) => Number(m.ts) > oldest); + const start = page * limit, end = Math.min(start + limit, filtered.length); + if (failOnce(url.pathname, page + 1, res)) return; + const next = nextCursor(page, end, filtered.length); + const body = {ok: true, messages: filtered.slice(start, end)}; + if (next) { body.has_more = true; body.response_metadata = {next_cursor: next}; } + else if (process.env.FAKE_HAS_MORE_NO_CURSOR) body.has_more = true; + return reply(res, body); + } + if (url.pathname === "/slack/conversations.replies") { + const thread = slackThreads().find((t) => t.ts === url.searchParams.get("ts")); + if (!thread) return reply(res, {ok: false, error: "thread_not_found"}); + const oldest = Number(url.searchParams.get("oldest")); + const all = [thread, ...thread.replies.filter((m) => Number(m.ts) > oldest) + .sort((a, b) => Number(a.ts) - Number(b.ts))]; + const page = pageNumber(url.searchParams.get("cursor")); + const limit = Number(url.searchParams.get("limit") || all.length); + const start = page * limit, end = Math.min(start + limit, all.length); + if (failOnce(url.pathname, page + 1, res)) return; + const next = nextCursor(page, end, all.length); + const body = {ok: true, messages: all.slice(start, end)}; + if (next) { body.has_more = true; body.response_metadata = {next_cursor: next}; } + else if (process.env.FAKE_HAS_MORE_NO_CURSOR) body.has_more = true; + return reply(res, body); + } + if (url.pathname === "/slack/users.info") return reply(res, {ok: true, user: {real_name: "Dana"}}); + if (url.pathname === "/gmail/messages") { + const all = Array.from({length: gmailCount}, (_, i) => ({id: `m${i + 1}`})); + const page = pageNumber(url.searchParams.get("pageToken")); + const limit = Number(url.searchParams.get("maxResults") || all.length); + const start = page * limit, end = Math.min(start + limit, all.length); + if (failOnce(url.pathname, page + 1, res)) return; + const next = nextCursor(page, end, all.length); + const body = {messages: all.slice(start, end)}; + if (next) body.nextPageToken = next; + return reply(res, body); + } + if (url.pathname.startsWith("/gmail/messages/")) { + const id = url.pathname.split("/").pop(); + const index = Number(id.slice(1)) - 1; + return reply(res, {id, internalDate: String((day * 1000) + 5000 + (index * 1000)), + snippet: `Message ${index + 1}`, labelIds: ["UNREAD"], + payload: {headers: [{name: "From", value: "cfo@example.com"}, + {name: "Subject", value: index === 0 ? "Q3" : `Mail ${index + 1}`}]}}); + } + res.writeHead(404); res.end(); +}; +// FAKE_TLS_DIR: serve HTTPS with that directory's self-signed cert (the +// stand-in for the sandbox egress proxy, which re-signs TLS with its own CA). +const tls = process.env.FAKE_TLS_DIR; +(tls ? https.createServer({ key: readFileSync(`${tls}/key.pem`), cert: readFileSync(`${tls}/cert.pem`) }, handler) + : http.createServer(handler)) + .listen(0, "127.0.0.1", function () { console.log(this.address().port); }); diff --git a/tests/fixtures/container-site.yaml b/tests/fixtures/container-site.yaml new file mode 100644 index 000000000..65331a2b3 --- /dev/null +++ b/tests/fixtures/container-site.yaml @@ -0,0 +1,404 @@ +# Synthetic chart inputs; not an installation configuration. +portal: + deploymentModes: + - vm + - container + containerSites: + alice: + modelLifecycle: true + governanceNamespace: saw-builds + subject: 11111111-1111-4111-8111-111111111111 + stack: + identity: + revision: '1111111111111111111111111111111111111111' + values: + enabled: true + ownerID: '@INSTALLATION@' + targetNamespace: '@RUNTIME@' + gateway: '@OWNER@' + serverSans: + - '@OWNER@-gateway.@RUNTIME@.svc' + - '@OWNER@-gateway.@RUNTIME@.svc.cluster.local' + policy: + revision: '1111111111111111111111111111111111111111' + values: + enabled: true + targetNamespace: '@RUNTIME@' + gatewayId: '@OWNER@' + gatewayServiceAccount: '@OWNER@-gateway' + sandboxServiceAccount: '@OWNER@-sandbox' + mode: container + network: + priority: 51 + egress: + - name: dns + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + podSelector: + matchLabels: + dns.operator.openshift.io/daemonset-dns: default + ports: + - portNumber: + protocol: UDP + port: 53 + - portNumber: + protocol: TCP + port: 53 + - portNumber: + protocol: UDP + port: 5353 + - portNumber: + protocol: TCP + port: 5353 + - name: gateway + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '@RUNTIME@' + podSelector: + matchLabels: + app.kubernetes.io/instance: '@OWNER@' + openshell.pattern/role: gateway + ports: + - portNumber: + protocol: TCP + port: 17670 + - name: workload-boundary + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '@RUNTIME@' + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + ports: + - portNumber: + protocol: TCP + port: 5500 + - name: credential-proxy + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '@TRUSTED@' + podSelector: + matchLabels: + app.kubernetes.io/instance: '@OWNER@' + openshell.pattern/role: credential-proxy + ports: + - portNumber: + protocol: TCP + port: 3128 + proxy: + revision: '1111111111111111111111111111111111111111' + values: + enabled: true + ownerID: '@INSTALLATION@' + gateway: '@OWNER@' + targetNamespace: '@RUNTIME@' + policyConfigMap: '@OWNER@-proxy-policy' + policyDigest: 3601c933b11c60e7d693c4635ed14c626e32c29c9a9e8920a144f852dcba184d + credentialSecretNames: + - inference + network: + bootstrapEgress: + - to: + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 6443 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + podSelector: + matchLabels: + dns.operator.openshift.io/daemonset-dns: default + ports: + - protocol: UDP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 53 + - protocol: TCP + port: 5353 + proxyEgress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + podSelector: + matchLabels: + dns.operator.openshift.io/daemonset-dns: default + ports: + - protocol: UDP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 53 + - protocol: TCP + port: 5353 + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 0.0.0.0/8 + - 10.0.0.0/8 + - 100.64.0.0/10 + - 127.0.0.0/8 + - 169.254.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 224.0.0.0/4 + - 240.0.0.0/4 + - 192.0.2.0/24 + ports: + - protocol: TCP + port: 443 + gateway: + revision: '1111111111111111111111111111111111111111' + values: + deployment: + mode: container + kubernetes: + experimental: true + workload: + image: registry.access.redhat.com/ubi9/python-311@sha256:a0bdb55576fc5b8d6704279307817828ef027e1065533ceba133fe9516003a6c + identity: + serverTlsSecret: '@OWNER@-server' + clientCaSecret: '@OWNER@-client-ca' + jwtSecret: '@OWNER@-jwt' + credentialKeySecret: '@OWNER@-kek' + storage: + className: block + gatewaySize: 1Gi + workspaceSize: 1Gi + proxy: + url: http://@OWNER@-credential-proxy.@TRUSTED@.svc:3128 + caConfigMap: '@OWNER@-proxy-ca' + governance: + endpoint: http://governance-interceptor.saw-builds.svc:18081 + network: + reconcilerNamespace: '@TRUSTED@' + reconcilerLabels: + app.kubernetes.io/instance: '@OWNER@' + openshell.pattern/role: reconciler + gatewayEgress: + - to: + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 6443 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + podSelector: + matchLabels: + dns.operator.openshift.io/daemonset-dns: default + ports: + - protocol: UDP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 53 + - protocol: TCP + port: 5353 + - to: + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + ports: + - protocol: TCP + port: 443 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: saw-builds + podSelector: + matchLabels: + app.kubernetes.io/name: governance-interceptor + ports: + - protocol: TCP + port: 18081 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '@TRUSTED@' + podSelector: + matchLabels: + app.kubernetes.io/instance: '@OWNER@' + openshell.pattern/role: credential-proxy + ports: + - protocol: TCP + port: 3128 + oidc: + issuerUrl: https://openshell-keycloak-ingress-saw-keycloak.apps.@DOMAIN@/realms/openshell + route: + enabled: false + audit: + revision: '1111111111111111111111111111111111111111' + values: + enabled: true + trustedNamespace: '@ARCHIVE@' + runtimeNamespace: '@RUNTIME@' + ownerID: '@INSTALLATION@' + storageClass: block-immediate + exchangeStorageClass: shared-filesystem + image: ghcr.io/openclaw/openclaw@sha256:0a5ff5e682e62afa19149df126aa50063bf65ef885b5c94713ce32dc0eb12e15 + identity: + subject: '@SUBJECT@' + username: '@OWNER@' + workspace: agent + sandbox: audited + automaticEnrollment: + image: registry.example.test/saw-runtime-controller@sha256:966c76c969f074ea088e1499d40fd8d993ecb4df527239736df554d9d726c6d0 + revision: '1111111111111111111111111111111111111111' + buildNamespace: saw-builds + resources: + limits: + cpu: '2' + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi + reference: + validation: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + binding: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + network: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + deny: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + podDeleteGuard: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + podDeleteBinding: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + secretGuard: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + secretBinding: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + serviceGuard: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + serviceBinding: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + gatewayRole: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + workloadFence: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + supervisorFence: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + gatewayNetwork: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + gateway: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + gatewayService: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + gatewayConfig: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + governance: + uid: 22222222-2222-4222-8222-222222222222 + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + governanceService: + uid: 22222222-2222-4222-8222-222222222222 + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + governancePolicy: + uid: 22222222-2222-4222-8222-222222222222 + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + governanceProfiles: + uid: 22222222-2222-4222-8222-222222222222 + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + proxy: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + proxyNetwork: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + proxyPolicy: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + publicCA: equals independent trusted proxy public CA + relay: + keycloakNamespace: saw-keycloak + keycloakProvisionerSecret: saw-relay-provisioner + issuer: https://openshell-keycloak-ingress-saw-keycloak.apps.cluster.example.test/realms/openshell + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + podSelector: + matchLabels: + dns.operator.openshift.io/daemonset-dns: default + ports: + - protocol: UDP + port: 53 + - protocol: UDP + port: 5353 + - protocol: TCP + port: 53 + - protocol: TCP + port: 5353 + - to: + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + - ipBlock: + cidr: 192.0.2.10/32 + ports: + - protocol: TCP + port: 443 + revision: 661194fd49c75b4e7dd202d4d2ad5fa7b04934e3 + approvedProfiles: + data-science: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + custom-inference: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + profileImageOverrides: + data-science: + notebook: registry.example.test/saw-openclaw-home@sha256:a89f54d5ffe6d1ee9df9c9a65978c26c54fd46aaf5f0b4e4c2c2c0c64ad5b18a + imagePull: + namespace: saw-builds + streams: + - saw-openclaw-home + targets: + default/notebook: + workspace: default + sandbox: notebook + ui: true + base: alice + relayID: a8cb6cbd0eb62f31 + subjectMap: saw-portal-relay-subject + selectedDigest: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + providerAllowHosts: + - api.openai.com + providerPolicy: + allowHosts: + - api.openai.com + bindings: + - secret: inference + key: api_key + providers: + - openai + markerPrefix: saw-cred + profiles: + openai: + - api.openai.com diff --git a/tests/fixtures/profiles/audited-openclaw/agent/providers.yaml b/tests/fixtures/profiles/audited-openclaw/agent/providers.yaml new file mode 100644 index 000000000..8e93fc03e --- /dev/null +++ b/tests/fixtures/profiles/audited-openclaw/agent/providers.yaml @@ -0,0 +1,8 @@ +apiVersion: saw.redhat.com/v1alpha1 +kind: Providers +spec: + providers: + - name: openai + type: openai + credentialSecret: inference + credentialSecretKey: api_key diff --git a/tests/fixtures/profiles/audited-openclaw/agent/sandbox.yaml b/tests/fixtures/profiles/audited-openclaw/agent/sandbox.yaml new file mode 100644 index 000000000..23b0cce96 --- /dev/null +++ b/tests/fixtures/profiles/audited-openclaw/agent/sandbox.yaml @@ -0,0 +1,14 @@ +apiVersion: saw.redhat.com/v1alpha1 +kind: Sandboxes +spec: + sandboxes: + - name: audited + type: openclaw + image: ghcr.io/openclaw/openclaw@sha256:0a5ff5e682e62afa19149df126aa50063bf65ef885b5c94713ce32dc0eb12e15 + providers: [openai] + inference: + providerRef: openai + protocol: openai + baseUrl: https://api.openai.com/v1 + model: gpt-4.1-mini-2025-04-14 + ui: {route: true} diff --git a/tests/fixtures/profiles/audited-openclaw/agent/workspace.yaml b/tests/fixtures/profiles/audited-openclaw/agent/workspace.yaml new file mode 100644 index 000000000..3b62e3fff --- /dev/null +++ b/tests/fixtures/profiles/audited-openclaw/agent/workspace.yaml @@ -0,0 +1,5 @@ +apiVersion: saw.redhat.com/v1alpha1 +kind: Workspace +metadata: + name: agent + description: OpenClaw chat with native tool approval and retained audit history diff --git a/tests/fixtures/saw-kubernetes-controller-values.yaml b/tests/fixtures/saw-kubernetes-controller-values.yaml new file mode 100644 index 000000000..a520eb5df --- /dev/null +++ b/tests/fixtures/saw-kubernetes-controller-values.yaml @@ -0,0 +1,22 @@ +# Synthetic public boundary inputs for offline chart validation. +enabled: true +image: ghcr.io/validatedpatterns-sandbox/saw-kubernetes-runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +stateClaim: portal-ledger +claimNames: [workspace-alice, saw-gate-alice, saw-audit-alice] +configuration: + gateway: alice + namespace: saw-alice + trustedNamespace: saw-ci-trusted + governanceNamespace: saw-governance + proxyPolicy: alice-proxy-policy + ownerID: 802fe583-9329-54b6-b5b4-7be58ec171f3 + namespaceUID: 11111111-1111-4111-8111-111111111111 + mode: container + intent: saw-runtime-intent + profile: saw-bom-profiles + status: saw-runtime-status + approved: + validation: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +trustedEgress: + - to: [{ipBlock: {cidr: 192.0.2.10/32}}] + ports: [{protocol: TCP, port: 6443}] diff --git a/tests/fixtures/saw-kubernetes-identity-values.yaml b/tests/fixtures/saw-kubernetes-identity-values.yaml new file mode 100644 index 000000000..ad6504a2d --- /dev/null +++ b/tests/fixtures/saw-kubernetes-identity-values.yaml @@ -0,0 +1,8 @@ +# Synthetic nonsecret render inputs; never a live installation. +enabled: true +ownerID: 67a9891d-e281-442f-86c3-41771f5ce049 +targetNamespace: saw-lint +gateway: saw-lint +serverSans: + - saw-lint-gateway.saw-lint.svc + - saw-lint-gateway.saw-lint.svc.cluster.local diff --git a/tests/fixtures/saw-kubernetes-values.yaml b/tests/fixtures/saw-kubernetes-values.yaml new file mode 100644 index 000000000..faee51e33 --- /dev/null +++ b/tests/fixtures/saw-kubernetes-values.yaml @@ -0,0 +1,32 @@ +# Synthetic offline values, not a live deployment or published artifact. +deployment: + mode: container +kubernetes: + experimental: true + workload: + image: registry.example.test/agent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + identity: + serverTlsSecret: gateway-server + clientCaSecret: gateway-client-ca + jwtSecret: gateway-jwt + credentialKeySecret: gateway-kek + storage: + className: test-rbd + proxy: + url: http://saw-test-credential-proxy.saw-test-trusted.svc:3128 + caConfigMap: saw-proxy-ca + governance: + endpoint: http://governance-interceptor.saw-governance.svc:18081 + network: + reconcilerNamespace: saw-test-trusted + reconcilerLabels: + openshell.pattern/role: reconciler + gatewayEgress: + - to: + - ipBlock: + cidr: 192.0.2.10/32 + ports: + - protocol: TCP + port: 6443 +oidc: + issuerUrl: https://sso.example.test/realms/openshell diff --git a/tests/fixtures/saw-workspace-relay-values.yaml b/tests/fixtures/saw-workspace-relay-values.yaml new file mode 100644 index 000000000..f11a4e9da --- /dev/null +++ b/tests/fixtures/saw-workspace-relay-values.yaml @@ -0,0 +1,22 @@ +# Synthetic owner-login and membership inputs; no credential values. +enabled: true +clientID: saw-alice +keycloakNamespace: saw-keycloak +keycloakProvisionerSecret: keycloak-admin +issuer: https://sso.example.test/realms/openshell +trustedNamespace: saw-ci-trusted +gatewayNamespace: saw-alice +gateway: alice +gatewayEndpoint: https://alice-gateway.saw-alice.svc:17670 +installationID: 802fe583-9329-54b6-b5b4-7be58ec171f3 +workspace: default +workspaceID: 33333333-3333-4333-8333-333333333333 +sandbox: notebook +ownerLogin: alice +ownerSubject: 11111111-1111-4111-8111-111111111111 +subject: 22222222-2222-4222-8222-222222222222 +sdkImage: ghcr.io/validatedpatterns-sandbox/saw-kubernetes-runtime@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +host: alice.apps.example.test +egress: + - to: [{ipBlock: {cidr: 192.0.2.10/32}}] + ports: [{protocol: TCP, port: 443}] diff --git a/tests/installer/conftest.py b/tests/installer/conftest.py index 9c30d71c2..d287fea78 100644 --- a/tests/installer/conftest.py +++ b/tests/installer/conftest.py @@ -439,6 +439,9 @@ def inputs_dir(tmp_path, bom, config, shipped_profile_files, secrets_dir, fake_e (installer / "installer-bom.yaml").write_text(yaml.safe_dump(bom)) (installer / "config.json").write_text(json.dumps(config)) (installer / "apply_bom.py").write_text(SCRIPT.read_text()) + (installer / "saw_core.py").write_text((SCRIPT.parent / "saw_core.py").read_text()) + (installer / "saw_admin.py").write_text((SCRIPT.parent / "saw_admin.py").read_text()) + (installer / "saw_ui_limit.py").write_text((SCRIPT.parent / "saw_ui_limit.py").read_text()) # Match the installer disk: typed endpoint validation uses this shipped # host catalog to distinguish built-in and deployment-owned profiles. for profile in (CHART / "files/provider-profiles").glob("*.yaml"): diff --git a/tests/installer/fakes/openshell b/tests/installer/fakes/openshell index 2aa1bab3c..986f2ad0a 100755 --- a/tests/installer/fakes/openshell +++ b/tests/installer/fakes/openshell @@ -21,7 +21,8 @@ def archive_bytes(content): raw = io.BytesIO() with tarfile.open(fileobj=raw, mode="w") as archive: for name, value in sorted(content.items()): - data = value.encode() + # Keep arbitrary archived bytes round-trippable through JSON state. + data = value.encode("utf-8", errors="surrogateescape") member = tarfile.TarInfo(name) member.uid = member.gid = 1000 member.mode, member.size = 0o644, len(data) @@ -395,8 +396,15 @@ elif op == "sandbox upload": if not remote.startswith("/sandbox/"): fail(f"Error: upload destination {remote!r} escapes the sandbox working directory") with tarfile.open(local, "r:gz") as archive: - content = {member.name.removeprefix("./"): archive.extractfile(member).read().decode() - for member in archive if member.isfile()} + content = {} + for member in archive: + name = member.name.removeprefix("./") + # The fake models the Linux OpenShell guest. macOS tar can add + # AppleDouble resource-fork entries for files in the host fixture. + if not member.isfile() or name.startswith("__MACOSX/") or Path(name).name.startswith("._"): + continue + content[name] = archive.extractfile(member).read().decode( + "utf-8", errors="surrogateescape") sb.setdefault("archives", {})[remote] = content sb.setdefault("archiveDigests", {})[remote] = hashlib.sha256(Path(local).read_bytes()).hexdigest() elif op == "sandbox delete": diff --git a/tests/installer/test_apply_profiles.py b/tests/installer/test_apply_profiles.py index a64f89b11..4ed21f57f 100644 --- a/tests/installer/test_apply_profiles.py +++ b/tests/installer/test_apply_profiles.py @@ -989,10 +989,10 @@ def test_confinement_repair_resumes_existing_agent_without_usable_provider( if provider_mode == "missing": monkeypatch.setattr(retry, "find_provider", lambda *args: None) else: - monkeypatch.setattr(ab, "sandbox_endpoint", lambda *args: "") - monkeypatch.setattr(ab, "NATIVE_BASE_URLS", {}) - monkeypatch.setattr(ab, "PROTOCOL_NATIVE_BASE_URLS", {}) - monkeypatch.setattr(ab, "OPENCLAW_AUTH_CHOICES", {}) + monkeypatch.setattr(ab._saw_core, "sandbox_endpoint", lambda *args: "") + monkeypatch.setattr(ab._saw_core, "NATIVE_BASE_URLS", {}) + monkeypatch.setattr(ab._saw_core, "PROTOCOL_NATIVE_BASE_URLS", {}) + monkeypatch.setattr(ab._saw_core, "OPENCLAW_AUTH_CHOICES", {}) before = len(fake_env.openshell_calls()) retry.apply(profiles) sb = fake_env.openshell_state()["sandboxes"][target] diff --git a/tests/installer/test_cli.py b/tests/installer/test_cli.py index a37a99208..8356e62f1 100644 --- a/tests/installer/test_cli.py +++ b/tests/installer/test_cli.py @@ -82,6 +82,12 @@ def test_install_then_apply_reaches_ready(world, credential_mode): copy = world.state / "installer" / "apply_bom.py" assert copy.read_text() == (world.inputs / "installer" / "apply_bom.py").read_text() assert oct(copy.stat().st_mode & 0o777) == "0o644" + # Raw source fixtures carry the exact shared source beside the copy. The + # signed chart-rendered installer embeds it instead (packaging tests). + for name in ("saw_core.py", "saw_admin.py", "saw_ui_limit.py"): + shared_copy = copy.with_name(name) + assert shared_copy.read_bytes() == (world.inputs / "installer" / name).read_bytes() + assert oct(shared_copy.stat().st_mode & 0o777) == "0o644" # Metadata-only inputs produce markers, and argv names only credential variables. credential_args = [call[i + 1] for call in world.fake.openshell_calls() for i, arg in enumerate(call[:-1]) if arg == "--credential"] diff --git a/tests/installer/test_daily_briefing.py b/tests/installer/test_daily_briefing.py index e082c8d25..035c715c4 100644 --- a/tests/installer/test_daily_briefing.py +++ b/tests/installer/test_daily_briefing.py @@ -174,135 +174,7 @@ def test_the_skill_schedules_with_the_openclaw_cron_command(): assert "use the `cron`" not in skill -FAKE_APIS = r''' -import { readFileSync, writeFileSync } from "node:fs"; -import http from "node:http"; -import https from "node:https"; -const day = Math.floor(new Date(new Date().setHours(0, 0, 0, 0)).getTime() / 1000); -const gmailCount = Number(process.env.FAKE_GMAIL_COUNT || 1); -const slackCount = Number(process.env.FAKE_SLACK_COUNT || 1); -const channelPages = Number(process.env.FAKE_CHANNEL_PAGES || 1); -const slackThreads = () => process.env.FAKE_SLACK_THREADS - ? JSON.parse(readFileSync(process.env.FAKE_SLACK_THREADS, "utf8")) : []; -const failSpec = process.env.FAKE_FAIL_ONCE || ""; -const failed = new Set(); -const requests = []; -const reply = (res, body, status = 200) => { - const send = () => { - res.writeHead(status, {"content-type": "application/json"}); - res.end(JSON.stringify(body)); - }; - const delay = Number(process.env.FAKE_DELAY_MS || 0); - if (delay > 0) setTimeout(send, delay); - else send(); -}; -const record = (url) => { - requests.push({path: url.pathname, query: Object.fromEntries(url.searchParams)}); - if (process.env.FAKE_REQUEST_LOG) writeFileSync(process.env.FAKE_REQUEST_LOG, JSON.stringify(requests)); -}; -const pageNumber = (cursor) => cursor ? Number(cursor) : 0; -const nextCursor = (page, end, count) => { - if (process.env.FAKE_REPEAT_CURSOR) return "repeat"; - return end < count ? String(page + 1) : ""; -}; -const failOnce = (path, page, res) => { - const key = `${path}:${page}`; - if (failSpec === key && !failed.has(key)) { - failed.add(key); - reply(res, {ok: false, error: "server_error"}, 500); - return true; - } - return false; -}; -const handler = (req, res) => { - const url = new URL(req.url, "http://x"); - record(url); - const expected = url.pathname.startsWith("/slack") ? "slack-ph" : "gmail-ph"; - if (req.headers.authorization !== `Bearer ${expected}`) - return reply(res, {ok: false, error: "invalid_auth"}, 401); - if (url.pathname === "/slack/conversations.list") { - const all = channelPages > 1 - ? [{id: "C0", name: "random", is_member: false}, {id: "C1", name: "eng", is_member: true}] - : [{id: "C1", name: "eng", is_member: true}, {id: "C2", name: "random", is_member: false}]; - const page = pageNumber(url.searchParams.get("cursor")); - const limit = Number(url.searchParams.get("limit") || all.length); - const start = page * limit, end = Math.min(start + limit, all.length); - if (failOnce(url.pathname, page + 1, res)) return; - const next = nextCursor(page, end, all.length); - const body = {ok: true, channels: all.slice(start, end)}; - if (next) { body.has_more = true; body.response_metadata = {next_cursor: next}; } - else if (process.env.FAKE_HAS_MORE_NO_CURSOR) body.has_more = true; - return reply(res, body); - } - if (url.pathname === "/slack/conversations.history") { - const oldest = Number(url.searchParams.get("oldest")); - const generated = Array.from({length: slackCount}, (_, i) => ({ - ts: String(day + 60 + i), user: "U1", - text: i === 0 ? "Can you review the PR today?" : `Slack message ${i + 1}`, - })); - const threaded = slackThreads().flatMap((thread) => [ - {...thread, thread_ts: thread.ts, reply_count: thread.replies.length, - latest_reply: thread.replies.reduce((latest, m) => Number(m.ts) > Number(latest) ? m.ts : latest, "0")}, - ...thread.replies.filter((m) => m.subtype === "thread_broadcast") - .map((m) => ({...m, thread_ts: thread.ts})), - ]); - const all = [...generated, ...threaded].sort((a, b) => Number(b.ts) - Number(a.ts)); - const page = pageNumber(url.searchParams.get("cursor")); - const limit = Number(url.searchParams.get("limit") || all.length); - const filtered = all.filter((m) => Number(m.ts) > oldest); - const start = page * limit, end = Math.min(start + limit, filtered.length); - if (failOnce(url.pathname, page + 1, res)) return; - const next = nextCursor(page, end, filtered.length); - const body = {ok: true, messages: filtered.slice(start, end)}; - if (next) { body.has_more = true; body.response_metadata = {next_cursor: next}; } - else if (process.env.FAKE_HAS_MORE_NO_CURSOR) body.has_more = true; - return reply(res, body); - } - if (url.pathname === "/slack/conversations.replies") { - const thread = slackThreads().find((t) => t.ts === url.searchParams.get("ts")); - if (!thread) return reply(res, {ok: false, error: "thread_not_found"}); - const oldest = Number(url.searchParams.get("oldest")); - const all = [thread, ...thread.replies.filter((m) => Number(m.ts) > oldest) - .sort((a, b) => Number(a.ts) - Number(b.ts))]; - const page = pageNumber(url.searchParams.get("cursor")); - const limit = Number(url.searchParams.get("limit") || all.length); - const start = page * limit, end = Math.min(start + limit, all.length); - if (failOnce(url.pathname, page + 1, res)) return; - const next = nextCursor(page, end, all.length); - const body = {ok: true, messages: all.slice(start, end)}; - if (next) { body.has_more = true; body.response_metadata = {next_cursor: next}; } - else if (process.env.FAKE_HAS_MORE_NO_CURSOR) body.has_more = true; - return reply(res, body); - } - if (url.pathname === "/slack/users.info") return reply(res, {ok: true, user: {real_name: "Dana"}}); - if (url.pathname === "/gmail/messages") { - const all = Array.from({length: gmailCount}, (_, i) => ({id: `m${i + 1}`})); - const page = pageNumber(url.searchParams.get("pageToken")); - const limit = Number(url.searchParams.get("maxResults") || all.length); - const start = page * limit, end = Math.min(start + limit, all.length); - if (failOnce(url.pathname, page + 1, res)) return; - const next = nextCursor(page, end, all.length); - const body = {messages: all.slice(start, end)}; - if (next) body.nextPageToken = next; - return reply(res, body); - } - if (url.pathname.startsWith("/gmail/messages/")) { - const id = url.pathname.split("/").pop(); - const index = Number(id.slice(1)) - 1; - return reply(res, {id, internalDate: String((day * 1000) + 5000 + (index * 1000)), - snippet: `Message ${index + 1}`, labelIds: ["UNREAD"], - payload: {headers: [{name: "From", value: "cfo@example.com"}, - {name: "Subject", value: index === 0 ? "Q3" : `Mail ${index + 1}`}]}}); - } - res.writeHead(404); res.end(); -}; -// FAKE_TLS_DIR: serve HTTPS with that directory's self-signed cert (the -// stand-in for the sandbox egress proxy, which re-signs TLS with its own CA). -const tls = process.env.FAKE_TLS_DIR; -(tls ? https.createServer({ key: readFileSync(`${tls}/key.pem`), cert: readFileSync(`${tls}/cert.pem`) }, handler) - : http.createServer(handler)) - .listen(0, "127.0.0.1", function () { console.log(this.address().port); }); -''' +FAKE_APIS = (ROOT / "tests/fixtures/briefing-apis.mjs").read_text() def mcp_call(server, env, tool, arguments=None): diff --git a/tests/installer/test_official_api_providers.py b/tests/installer/test_official_api_providers.py index e3f60f600..5c0d7bfc3 100644 --- a/tests/installer/test_official_api_providers.py +++ b/tests/installer/test_official_api_providers.py @@ -200,6 +200,7 @@ def test_existing_custom_settings_are_cleared_only_for_native_apis( pytest.param("legacy", "nvidia", "", "custom-api-key", "nvidia/m", "openai", id="legacy-nvidia"), pytest.param("typed", "nvidia", "", "custom-api-key", "nvidia/m", "openai", id="typed-nvidia"), pytest.param("typed", "openai", "", "openai-api-key", "gpt-4o", "openai", id="typed-openai-native"), + pytest.param("typed", "openai", "https://api.openai.com/v1", "custom-api-key", "gpt-4.1-mini", "openai", id="typed-openai-explicit-official-model"), pytest.param("typed", "anthropic", "https://gateway.example/anthropic", "custom-api-key", "claude-sonnet-4-6", "anthropic", id="typed-anthropic-custom"), ]) def test_onboarding_uses_the_provider_protocol( diff --git a/tests/installer/test_shared_core_packaging.py b/tests/installer/test_shared_core_packaging.py new file mode 100644 index 000000000..7705b7cd8 --- /dev/null +++ b/tests/installer/test_shared_core_packaging.py @@ -0,0 +1,279 @@ +"""Embedded core stays inside the existing signed/staged installer artifact. + +No installer code is imported by the signing tool. The legacy manifest format +is deliberately independent here: old golden-image verifiers must cover exactly +the same bytes without acquiring a new module or a new verifier. +""" + +import base64 +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +CHART = ROOT / "charts/openshell-saw" +CORE = CHART / "files/installer/saw_core.py" +APPLY = CHART / "files/installer/apply_bom.py" +HELM = shutil.which("helm") +MARKER = '_SAW_CORE_EMBEDDED_B64 = "__SAW_CORE_SOURCE_BASE64__"' +LEGACY_FIXED = ("installer-bom.yaml", "apply_bom.py", "setup-dashboard.sh", + "tool-actions.yaml", "tool-gate.mjs", "tool-gate-plugin.mjs", + "saw-gateway-pki", "openshell-gateway-pki.conf") + + +@pytest.fixture(scope="module") +def signer(): + spec = importlib.util.spec_from_file_location("shared_core_manifest", ROOT / "scripts/build-installer-manifest.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def helm_template(chart=CHART): + return subprocess.run([ + HELM, "template", "saw-test", str(chart), "--namespace", "saw-test", + "-f", str(ROOT / "tests/fixtures/saw-network-topology.yaml"), + "-f", str(ROOT / "tests/fixtures/credential-proxy-values.yaml"), + "--set", "sandboxName=saw-test", + ], capture_output=True, text=True) + + +@pytest.fixture(scope="module") +def rendered_data(): + if not HELM: + pytest.skip("helm is not installed") + result = helm_template() + assert result.returncode == 0, result.stderr + return next(doc["data"] for doc in yaml.safe_load_all(result.stdout) + if doc and doc["kind"] == "ConfigMap" and doc["metadata"]["name"] == "saw-test-installer") + + +def replace_core(source, data): + encoded = base64.b64encode(data).decode("ascii") + result, count = re.subn(r'^_SAW_CORE_EMBEDDED_B64 = .*$', + lambda _: f'_SAW_CORE_EMBEDDED_B64 = "{encoded}"', source, flags=re.MULTILINE) + assert count == 1 + return result + + +def legacy_manifest(data): + names = sorted((*LEGACY_FIXED, *(name for name in data + if name.startswith("provider-profile-") and name.endswith(".yaml")))) + return "".join(f'{hashlib.sha256(data[name].encode("utf-8")).hexdigest()} {name}\n' + for name in names) + + +def run_file(path, *, cwd=None, pythonpath=None): + env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} + if pythonpath is not None: + env["PYTHONPATH"] = str(pythonpath) + return subprocess.run([sys.executable, str(path), "--help"], cwd=cwd, env=env, + capture_output=True, text=True) + + +def test_render_embeds_exact_canonical_bytes_and_preserves_legacy_manifest(rendered_data, signer): + assert "saw_ui_limit.py" not in rendered_data + assert signer.embedded_core(rendered_data["apply_bom.py"], "_SAW_UI_LIMIT_EMBEDDED_B64") == CORE.with_name("saw_ui_limit.py").read_bytes() + assert "saw_admin.py" not in rendered_data + assert signer.embedded_core(rendered_data["apply_bom.py"], "_SAW_ADMIN_EMBEDDED_B64") == CORE.with_name("saw_admin.py").read_bytes() + assert "saw_core.py" not in rendered_data + assert MARKER not in rendered_data["apply_bom.py"] + assert signer.embedded_core(rendered_data["apply_bom.py"]) == CORE.read_bytes() + assert signer.build_manifest(rendered_data) == legacy_manifest(rendered_data) + assert set(signer.FIXED_FILES + signer.TOOL_GATE_FILES + signer.PKI_FILES) == set(LEGACY_FIXED) + + +@pytest.mark.parametrize("change", ["missing", "duplicate"]) +def test_chart_refuses_missing_or_duplicate_loader_marker(tmp_path, change): + if not HELM: + pytest.skip("helm is not installed") + chart = tmp_path / "chart" + shutil.copytree(CHART, chart) + path = chart / "files/installer/apply_bom.py" + source = path.read_text() + path.write_text(source.replace(MARKER, "" if change == "missing" else MARKER + "\n" + MARKER)) + result = helm_template(chart) + assert result.returncode != 0 + assert "exactly one shared-core embedding marker" in result.stderr + + +def test_chart_refuses_missing_canonical_core(tmp_path): + if not HELM: + pytest.skip("helm is not installed") + chart = tmp_path / "chart" + shutil.copytree(CHART, chart) + (chart / "files/installer/saw_core.py").unlink() + result = helm_template(chart) + assert result.returncode != 0 + assert "canonical files/installer/saw_core.py is required" in result.stderr + + +@pytest.mark.parametrize("source", [ + "print('no embedded core')\n", + MARKER + "\n", + '_SAW_CORE_EMBEDDED_B64 = ""\n', + '_SAW_CORE_EMBEDDED_B64 = "%%%"\n', + '_SAW_CORE_EMBEDDED_B64 = "YWJj"\n_SAW_CORE_EMBEDDED_B64 = "YWJj"\n', + '_SAW_CORE_EMBEDDED_B64 = __import__("saw_core").source\n', +]) +def test_signer_refuses_unrendered_missing_or_malformed_core(rendered_data, signer, source): + data = {**rendered_data, "apply_bom.py": source} + with pytest.raises(SystemExit, match="shared-core|embedding"): + signer.build_manifest(data) + + +def test_signer_refuses_noncanonical_core_and_unsigned_adjacent_module(rendered_data, signer): + data = {**rendered_data, "apply_bom.py": replace_core(rendered_data["apply_bom.py"], CORE.read_bytes() + b"\n# drift\n")} + with pytest.raises(SystemExit, match="canonical saw_core.py"): + signer.build_manifest(data) + with pytest.raises(SystemExit, match="unsigned adjacent module"): + signer.build_manifest({**rendered_data, "saw_core.py": CORE.read_text()}) + + +def test_signer_never_executes_installer_or_core_code(rendered_data, signer, tmp_path): + witness = tmp_path / "must-not-exist" + source = rendered_data["apply_bom.py"] + f'\nopen({str(witness)!r}, "w").write("executed")\n' + signer.build_manifest({**rendered_data, "apply_bom.py": source}) + assert not witness.exists() + + +def test_embedded_installer_and_runtime_copy_ignore_adjacent_and_pythonpath_core(rendered_data, tmp_path): + witness = tmp_path / "untrusted-import" + hostile = f'open({str(witness)!r}, "w").write("imported")\nraise RuntimeError("untrusted core")\n' + live, staged, user, cwd = (tmp_path / name for name in ("live", "verified", "runtime-user", "cwd")) + for directory in (live, staged, user, cwd): + directory.mkdir() + (directory / "saw_core.py").write_text(hostile) + (directory / "saw_admin.py").write_text(hostile) + (directory / "saw_ui_limit.py").write_text(hostile) + (live / "apply_bom.py").write_text(rendered_data["apply_bom.py"]) + # Golden-image staging and cmd_apply both copy the same complete artifact. + shutil.copyfile(live / "apply_bom.py", staged / "apply_bom.py") + shutil.copyfile(staged / "apply_bom.py", user / "apply_bom.py") + for directory in (live, staged, user): + result = run_file(directory / "apply_bom.py", cwd=cwd, pythonpath=cwd) + assert result.returncode == 0, result.stderr + assert "apply-profiles" in result.stdout + assert not witness.exists() + (user / "saw_core.py").unlink() + assert run_file(user / "apply_bom.py", cwd=cwd, pythonpath=cwd).returncode == 0 + + +@pytest.mark.parametrize("replacement", ["", '_SAW_CORE_EMBEDDED_B64 = "%%%"', '_SAW_CORE_EMBEDDED_B64 = ""']) +def test_missing_or_broken_embedded_payload_never_falls_back_to_adjacent_core(rendered_data, tmp_path, replacement): + witness = tmp_path / "untrusted-import" + (tmp_path / "saw_core.py").write_text(f'open({str(witness)!r}, "w").write("imported")\n') + source = re.sub(r'^_SAW_CORE_EMBEDDED_B64 = .*$', lambda _: replacement, + rendered_data["apply_bom.py"], flags=re.MULTILINE) + path = tmp_path / "apply_bom.py" + path.write_text(source) + assert run_file(path).returncode != 0 + assert not witness.exists() + + +def test_raw_source_with_missing_sibling_never_uses_cwd_or_pythonpath_core(tmp_path): + source, cwd = tmp_path / "source", tmp_path / "cwd" + source.mkdir() + cwd.mkdir() + shutil.copyfile(APPLY, source / "apply_bom.py") + witness = tmp_path / "untrusted-import" + (cwd / "saw_core.py").write_text(f'open({str(witness)!r}, "w").write("imported")\n') + result = run_file(source / "apply_bom.py", cwd=cwd, pythonpath=cwd) + assert result.returncode != 0 + assert not witness.exists() + + +@pytest.mark.skipif(not shutil.which("openssl"), reason="OpenSSL is required for signature test") +def test_legacy_signature_covers_embedded_core_bytes(rendered_data, tmp_path): + key, pub, payload, signature = (tmp_path / name for name in ("key.pem", "pub.pem", "manifest", "signature")) + subprocess.run(["openssl", "genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", + "-out", str(key)], check=True, capture_output=True) + subprocess.run(["openssl", "pkey", "-in", str(key), "-pubout", "-out", str(pub)], check=True, capture_output=True) + payload.write_text(legacy_manifest(rendered_data)) + subprocess.run(["openssl", "dgst", "-sha256", "-sign", str(key), "-out", str(signature), str(payload)], + check=True, capture_output=True) + verify = ["openssl", "dgst", "-sha256", "-verify", str(pub), "-signature", str(signature), str(payload)] + assert subprocess.run(verify, capture_output=True).returncode == 0 + changed = copy.deepcopy(rendered_data) + changed["apply_bom.py"] = replace_core(changed["apply_bom.py"], CORE.read_bytes() + b"\n# modified core\n") + payload.write_text(legacy_manifest(changed)) + assert subprocess.run(verify, capture_output=True).returncode != 0 + + +@pytest.mark.skipif(sys.platform != "linux", reason="golden-image stager uses Linux tools") +def test_golden_verifier_stages_embedded_core_and_refuses_tampering(rendered_data, tmp_path): + """Exercise the real shell verifier using the suite's OpenSSL cosign shim.""" + installer, trust, bindir = (tmp_path / name for name in ("installer", "trust", "bin")) + for path in (installer, trust, bindir): + path.mkdir() + for name, content in rendered_data.items(): + (installer / name).write_text(content) + (installer / "config.json").write_text(json.dumps({"signing": {"mode": "enforce"}})) + witness = tmp_path / "untrusted-import" + # An old verifier ignores this extra file. Signed embedded code must also + # ignore it, including after staging and before apply-profiles handoff. + (installer / "saw_core.py").write_text(f'open({str(witness)!r}, "w").write("imported")\n') + key, pub = trust / "test.key", trust / "test.pub" + subprocess.run(["openssl", "genpkey", "-algorithm", "EC", "-pkeyopt", "ec_paramgen_curve:P-256", + "-out", str(key)], check=True, capture_output=True) + subprocess.run(["openssl", "pkey", "-in", str(key), "-pubout", "-out", str(pub)], check=True, capture_output=True) + payload = tmp_path / "manifest" + payload.write_text(legacy_manifest(rendered_data)) + subprocess.run(["openssl", "dgst", "-sha256", "-sign", str(key), "-out", + str(installer / "bundle.sigstore.json"), str(payload)], check=True, capture_output=True) + cosign = bindir / "cosign" + cosign.write_text( + '#!/bin/bash\nkey= bundle= previous=\n' + 'for arg in "$@"; do\n' + ' [[ "$previous" == --key ]] && key=$arg\n' + ' [[ "$previous" == --bundle ]] && bundle=$arg\n' + ' previous=$arg\ndone\n' + 'payload=${@: -1}\n' + 'exec openssl dgst -sha256 -verify "$key" -signature "$bundle" "$payload"\n') + cosign.chmod(0o755) + status = tmp_path / "status.json" + stage = tmp_path / "verified" + env = {**os.environ, "PATH": f"{bindir}:{os.environ['PATH']}", + "SAW_INSTALLER_DIR": str(installer), "SAW_TRUST_DIR": str(trust), + "SAW_STATUS_FILE": str(status), "SAW_STAGE_ROOT": str(stage), + "SAW_SIGNING_FLOOR_FILE": str(tmp_path / "no-floor"), "SAW_TRUST_KEYS": "test"} + command = ["bash", str(CHART / "files/guest/verify-bundle")] + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert json.loads(status.read_text())["bundle"]["signature"] == "verified" + staged_apply = stage / "installer/apply_bom.py" + assert staged_apply.read_text() == rendered_data["apply_bom.py"] + assert run_file(staged_apply).returncode == 0 + assert not witness.exists() + (installer / "apply_bom.py").write_text(replace_core( + rendered_data["apply_bom.py"], CORE.read_bytes() + f'\nopen({str(witness)!r}, "w").write("tampered")\n'.encode())) + rejected = subprocess.run(command, env=env, capture_output=True, text=True) + assert rejected.returncode == 1 + assert json.loads(status.read_text())["bundle"]["signature"] == "failed" + assert staged_apply.read_text() == rendered_data["apply_bom.py"] + assert run_file(staged_apply).returncode == 0 + assert not witness.exists() + + +@pytest.mark.parametrize("name", ["admin", "ui_limit"]) +def test_shared_admin_tampering_and_unsigned_module_rejected(rendered_data, signer, name): + filename = "saw_" + name + ".py" + field = "_SAW_" + name.upper() + "_EMBEDDED_B64" + changed = base64.b64encode(CORE.with_name(filename).read_bytes() + b"\n# drift\n").decode() + source = re.sub(r'^' + field + r' = .*$', + lambda _: field + ' = "' + changed + '"', + rendered_data["apply_bom.py"], flags=re.MULTILINE) + with pytest.raises(SystemExit, match="canonical " + filename): + signer.build_manifest({**rendered_data, "apply_bom.py": source}) + with pytest.raises(SystemExit, match="signed artifact"): + signer.build_manifest({**rendered_data, filename: "foreign"}) diff --git a/tests/installer/test_shared_profiles.py b/tests/installer/test_shared_profiles.py new file mode 100644 index 000000000..d4c8d282d --- /dev/null +++ b/tests/installer/test_shared_profiles.py @@ -0,0 +1,196 @@ +"""One profile language and dependency sequence for VM and Kubernetes backends.""" + +from dataclasses import asdict +from types import SimpleNamespace + +import pytest +import yaml + +from conftest import PROFILES, profile_files + + +def files_for(*, providers=None, sandboxes=None, workspace=None): + documents = {"workspace.yaml": ("Workspace", workspace or {"enabled": True})} + if providers is not None: + documents["providers.yaml"] = ("Providers", {"providers": providers}) + if sandboxes is not None: + documents["sandbox.yaml"] = ("Sandboxes", {"sandboxes": sandboxes}) + return {"profiles__p__team__" + filename: yaml.safe_dump({ + "apiVersion": "saw.redhat.com/v1alpha1", "kind": kind, + "metadata": {"name": "team"}, "spec": spec}) + for filename, (kind, spec) in documents.items()} + + +@pytest.mark.parametrize("profile", sorted(p.name for p in PROFILES.iterdir() if p.is_dir())) +def test_every_shipped_profile_has_the_same_strict_and_legacy_normalization(ab, profile): + files = profile_files(profile) + strict = ab.parse_profiles(files, strict=True) + legacy = ab.parse_profiles(files) + assert [asdict(p) for p in strict] == [asdict(p) for p in legacy] + ab.validate_profiles(strict, governed=True) + for _, workspace in ab.enabled_workspaces(strict): + for provider in workspace.providers: + assert provider.credential_values == {} + assert provider.refresh_material == {} + for sandbox in workspace.sandboxes: + assert sandbox.inference_values == {} + + +def test_vm_reexports_actual_shared_objects(ab): + for name in ("Provider", "Sandbox", "Workspace", "Profile", "HarnessBundle", + "InstallerError", "read_profile_files", "parse_profiles", + "validate_profiles", "apply_workspace_sequence"): + assert getattr(ab, name) is getattr(ab._saw_core, name) + + +@pytest.mark.parametrize("files", [ + files_for(workspace={"enabled": "false"}), + files_for(workspace={"enabled": False, "serviceAccountName": "admin"}), + files_for(providers=[{"name": "p", "type": "p", "enabled": "false"}]), + files_for(providers=[{"name": "p", "type": "p", "credentials": {"api_key": {"value": "private"}}}]), + files_for(providers=[{"name": "p", "type": "p", "refresh": {"strategy": "oauth2_refresh_token", "token": "private"}}]), + files_for(sandboxes=[{"name": "s", "enabled": False, "privileged": True}]), + files_for(sandboxes=[{"name": "s", "providers": "p"}]), + files_for(sandboxes=[{"name": "s", "harnessRef": {"hostPath": "/"}}]), + files_for(sandboxes=[{"name": "s", "harnessRequired": "false"}]), + files_for(sandboxes=[{"name": "s", "ui": {"route": "false"}}]), + files_for(sandboxes=[{"name": "s", "ui": {"serviceAccountName": "admin"}}]), + files_for(sandboxes=[{"name": "s", "env": {"TEST": {"value": "ok", 1: "bad"}}}]), + files_for(sandboxes=[{"name": "s", "env": {"TEST": {"value": "ok", "title": False}}}]), + files_for(sandboxes=[{"name": "s", "inference": {"model": {"secretKeyRef": {"name": 1, "key": "model"}}}}]), + files_for(sandboxes=[{"name": "s", "inference": {"model": {"secretKeyRef": {"name": "i", "key": "model"}, 1: "bad"}}}]), + files_for(sandboxes=[{"name": "s", "inference": {"model": {"secretKeyRef": {"name": "i", "key": "model"}, "title": False}}}]), +]) +def test_strict_schema_rejects_unknown_privilege_and_mistyped_fields(ab, files): + with pytest.raises(ab.InstallerError): + ab.parse_profiles(files, strict=True) + + +def test_strict_profile_preserves_supported_inputs_without_resolving_secrets(ab, monkeypatch): + def forbid_read(*args, **kwargs): + raise AssertionError("pure parser must not read files or resolve credentials") + + monkeypatch.setattr(ab._saw_core.Path, "read_text", forbid_read) + provider = {"name": "openai", "type": "openai", "credentialSecret": "inference", + "credentials": {"api_key": {"secretKey": "token", "title": "Provider API key"}}, + "refresh": {"strategy": "oauth2_refresh_token"}} + sandbox = {"name": "agent", "type": "openclaw", "providers": ["openai"], + "env": {"RUN_MODE": {"value": "demo"}}, "harnessRef": {"name": "harness"}, + "inference": {"providerRef": "openai", "protocol": "openai", + "model": {"secretKeyRef": {"name": "inference", "key": "model"}}, + "timeoutSeconds": 300}} + profiles = ab.parse_profiles(files_for(providers=[provider], sandboxes=[sandbox]), strict=True) + ab.validate_profiles(profiles, governed=True) + workspace = profiles[0].workspaces[0] + assert workspace.providers[0].credentials_spec == provider["credentials"] + assert workspace.providers[0].credential_values == {} + assert workspace.providers[0].refresh_material == {} + assert workspace.sandboxes[0].inference == sandbox["inference"] + assert workspace.sandboxes[0].inference_values == {} + assert workspace.sandboxes[0].env_values == {"RUN_MODE": "demo"} + + +def test_legacy_parse_behavior_and_private_plan_roundtrip_remain_compatible(ab): + # Strict parsing is opt-in for new adapters; preserve the VM compatibility + # contract, including its separate private runtime-user transport plan. + files = files_for(sandboxes=[{"name": "s", "futureField": True}]) + profiles = ab.parse_profiles(files) + profiles[0].workspaces[0].providers.append(ab.Provider( + name="openai", type="openai", credential_values={"api_key": "synthetic-test-marker"})) + plan = {"profiles": [asdict(p) for p in profiles]} + assert ab.profiles_from_plan(plan) == profiles + with pytest.raises(ab.InstallerError): + ab.parse_profiles(files, strict=True) + + +def test_backend_document_check_sees_chart_directives_before_normalization(ab): + files = files_for(sandboxes=[{"name": "s", "ui": {"route": True}, "harnessRequired": True}]) + seen = [] + + def capabilities(doc, kind): + seen.append(kind) + if kind == "Sandboxes": + assert doc["spec"]["sandboxes"][0]["harnessRequired"] is True + if doc["spec"]["sandboxes"][0]["ui"]["route"]: + raise ab.InstallerError("backend has no UI route support") + + with pytest.raises(ab.InstallerError, match="backend has no UI route support"): + ab.parse_profiles(files, strict=True, document_check=capabilities) + assert seen == ["Workspace", "Sandboxes"] + + +def test_strict_schema_runs_before_backend_document_check(ab): + seen = [] + with pytest.raises(ab.InstallerError): + ab.parse_profiles(files_for(workspace={"privileged": True}), strict=True, + document_check=lambda *args: seen.append(args)) + assert seen == [] + + +@pytest.mark.parametrize("workspace_ready, pending_providers, pending_sandboxes, expected, ready", [ + (False, set(), set(), ["workspace"], False), + (True, {"p1"}, set(), ["workspace", "p1", "p2"], False), + (True, {"p2"}, set(), ["workspace", "p1", "p2"], False), + (True, set(), {"s1"}, ["workspace", "p1", "p2", "s1", "s2"], False), + (True, set(), set(), ["workspace", "p1", "p2", "s1", "s2"], True), +]) +def test_shared_sequence_preserves_dependency_barrier_and_visits_all_peers( + ab, workspace_ready, pending_providers, pending_sandboxes, expected, ready): + calls = [] + workspace = object() + + def prepare(ws): + assert ws is workspace + calls.append("workspace") + return workspace_ready + + def apply_provider(ws, provider): + assert ws is workspace + calls.append(provider) + return provider not in pending_providers + + def apply_sandbox(ws, sandbox): + assert ws is workspace + calls.append(sandbox) + return sandbox not in pending_sandboxes + + adapter = SimpleNamespace(prepare_workspace=prepare, apply_provider=apply_provider, + apply_sandbox=apply_sandbox) + assert ab.apply_workspace_sequence(workspace, ["p1", "p2"], ["s1", "s2"], adapter) is ready + assert calls == expected + + +def test_shared_sequence_propagates_backend_errors_before_dependents(ab): + calls = [] + + def fail(ws, provider): + calls.append(provider) + raise ab.InstallerError("provider refused") + + adapter = SimpleNamespace(prepare_workspace=lambda ws: True, apply_provider=fail, + apply_sandbox=lambda ws, sb: calls.append(sb)) + with pytest.raises(ab.InstallerError, match="provider refused"): + ab.apply_workspace_sequence("workspace", ["p1", "p2"], ["s1"], adapter) + assert calls == ["p1"] + + +def test_vm_sequence_keeps_migration_and_aggregated_sandbox_failures(ab): + calls = [] + + def failing_sandbox(ws, sb): + calls.append(sb.name) + if sb.name == "bad": + raise ab.InstallerError("synthetic failure") + + applier = SimpleNamespace( + apply_workspace=lambda ws: calls.append("workspace"), + adopt_governed_profiles=lambda ws: calls.append("catalog"), + prepare_credential_migration=lambda ws: calls.append("migration"), + apply_provider=lambda ws, p: calls.append(p.name), + apply_sandbox=failing_sandbox, sandbox_failures=[]) + adapter = ab._VMWorkspaceSequence(applier) + assert ab.apply_workspace_sequence( + ab.Workspace("team"), [ab.Provider("p", "p")], + [ab.Sandbox("bad"), ab.Sandbox("disabled", enabled=False), ab.Sandbox("good")], adapter) is False + assert calls == ["workspace", "catalog", "migration", "p", "bad", "good"] + assert applier.sandbox_failures == ["sandbox 'bad': synthetic failure"] diff --git a/tests/installer/test_shared_provider_selection.py b/tests/installer/test_shared_provider_selection.py new file mode 100644 index 000000000..21ad6c98a --- /dev/null +++ b/tests/installer/test_shared_provider_selection.py @@ -0,0 +1,107 @@ +"""Agent onboarding choices shared by VM and Kubernetes adapters.""" +from dataclasses import asdict + +import pytest + + +def test_selection_prefers_declared_model_without_crossing_attachment_boundary(ab): + unrelated = ab.Provider("unrelated", "openai", model="wrong") + first = ab.Provider("first", "openai") + second = ab.Provider("second", "nvidia", model="right") + providers = [unrelated, first, second] + before = [asdict(p) for p in providers] + assert ab.select_inference_provider(providers, ["first", "second"]) is second + assert ab.select_inference_provider(providers, ["first"]) is first + assert ab.select_inference_provider(providers, ["absent"]) is None + assert [asdict(p) for p in providers] == before + + +def test_attachment_order_wins_over_workspace_order(ab): + first = ab.Provider("first", "openai", model="one") + second = ab.Provider("second", "openai", model="two") + assert ab.select_inference_provider([first, second], ["second", "first"]) is second + first.model = second.model = "" + assert ab.select_inference_provider([first, second], ["second", "first"]) is second + + +def test_explicit_reference_never_falls_back(ab): + chosen = ab.Provider("chosen", "openai") + fallback = ab.Provider("fallback", "nvidia", model="other") + select = ab.select_inference_provider + assert select([fallback, chosen], ["fallback", "chosen"], inference_ref="chosen") is chosen + assert select([fallback], ["fallback", "chosen"], inference_ref="chosen") is None + assert select([fallback, chosen], ["fallback", "chosen"], inference_ref="missing") is None + + +@pytest.mark.parametrize("options", [{"enabled": False}, {"credential_free": True}]) +def test_unavailable_or_keyless_provider_cannot_onboard_agent(ab, options): + excluded = ab.Provider("excluded", "openai", model="must-not-use", **options) + allowed = ab.Provider("allowed", "nvidia") + select = ab.select_inference_provider + assert select([excluded, allowed], ["excluded", "allowed"]) is allowed + assert select([excluded, allowed], ["excluded", "allowed"], inference_ref="excluded") is None + assert select([excluded], []) is None + + +def test_legacy_empty_attachments_preserve_available_candidate_order(ab): + first = ab.Provider("first", "openai") + second = ab.Provider("second", "nvidia", model="legacy") + assert ab.select_inference_provider([first, second], []) is second + second.model = "" + assert ab.select_inference_provider([first, second], []) is first + assert ab.select_inference_provider([], []) is None + + +def test_vm_adapter_keeps_skipped_providers_unavailable(ab): + applier = object.__new__(ab.ProfileApplier) + applier.skipped = {("team", "chosen")} + chosen = ab.Provider("chosen", "openai", model="one") + fallback = ab.Provider("fallback", "nvidia", model="two") + ws = ab.Workspace("team", providers=[chosen, fallback]) + sb = ab.Sandbox("chat", providers=["chosen", "fallback"], + inference_values={"providerRef": "chosen"}) + assert applier.find_provider(ws, sb.providers, sb) is None + sb.inference_values = {} + assert applier.find_provider(ws, sb.providers, sb) is fallback + + +def test_vm_reexports_canonical_selection_and_onboarding_helpers(ab): + for name in ("select_inference_provider", "sandbox_protocol", "sandbox_model", + "sandbox_endpoint", "openclaw_inference_settings", "NATIVE_BASE_URLS", + "PROTOCOL_NATIVE_BASE_URLS", "OPENCLAW_AUTH_CHOICES"): + assert getattr(ab, name) is getattr(ab._saw_core, name) + + +@pytest.mark.parametrize("provider, inference, expected", [ + ({"type": "openai"}, {}, ("https://api.openai.com/v1", "openai", "openai-api-key")), + ({"type": "anthropic"}, {}, ("", "anthropic", "apiKey")), + ({"type": "nvidia"}, {}, ("https://integrate.api.nvidia.com/v1", "openai", None)), + ({"type": "openai", "base_url_secret_key": "url"}, {}, + ("https://api.openai.com/v1", "openai", None)), + ({"type": "openai"}, {"protocol": "openai", "baseUrl": ""}, + ("https://api.openai.com/v1", "openai", "openai-api-key")), + ({"type": "openai"}, {"protocol": "openai", "baseUrl": "https://api.openai.com/v1"}, + ("https://api.openai.com/v1", "openai", None)), + ({"type": "custom-inference"}, {"protocol": "openai", "baseUrl": ""}, + ("https://api.openai.com/v1", "openai", None)), + ({"type": "custom-inference"}, {"protocol": "anthropic", "baseUrl": "https://models.example/v1"}, + ("https://models.example/v1", "anthropic", None)), +]) +def test_onboarding_endpoint_and_native_auth_choices(ab, provider, inference, expected): + p = ab.Provider("inference", **provider) + sb = ab.Sandbox("chat", inference_values={"model": "resolved", **inference} if inference else {}) + settings = ab.openclaw_inference_settings(sb, p) + assert (settings["base_url"], settings["protocol"], settings["auth_choice"]) == expected + assert settings["model"] == ("resolved" if inference else "nvidia/nemotron-3.5-lightning-30b-a3b") + + +def test_resolved_model_and_endpoint_override_legacy_fields(ab): + provider = ab.Provider("inference", "openai", model="provider-model", base_url="https://legacy.example/v1") + sb = ab.Sandbox("chat", model="sandbox-model") + assert ab.sandbox_model(sb, provider) == "sandbox-model" + sb.model = "" + assert ab.sandbox_model(sb, provider) == "provider-model" + sb.inference_values = {"model": "resolved-model", "protocol": "openai", "baseUrl": "https://new.example/v1"} + settings = ab.openclaw_inference_settings(sb, provider) + assert settings == {"model": "resolved-model", "protocol": "openai", "base_url": "https://new.example/v1", "auth_choice": None} + assert provider.base_url == "https://legacy.example/v1" diff --git a/tests/installer/test_signing.py b/tests/installer/test_signing.py index 262b601c6..4ddc770e8 100644 --- a/tests/installer/test_signing.py +++ b/tests/installer/test_signing.py @@ -21,6 +21,7 @@ def _write_bundle_files(directory): def test_manifest_requires_every_tool_gate_file(): + import base64 import importlib.util from pathlib import Path script = Path(__file__).resolve().parents[2] / "scripts/build-installer-manifest.py" @@ -28,6 +29,12 @@ def test_manifest_requires_every_tool_gate_file(): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) data = {name: name + "\n" for name in BUNDLE_FILES} + data["apply_bom.py"] = '_SAW_CORE_EMBEDDED_B64 = "' + base64.b64encode( + module.CORE_SOURCE.read_bytes()).decode("ascii") + '"\n' + data["apply_bom.py"] += '_SAW_ADMIN_EMBEDDED_B64 = "' + base64.b64encode( + module.CORE_SOURCE.with_name("saw_admin.py").read_bytes()).decode("ascii") + '"\n' + data["apply_bom.py"] += '_SAW_UI_LIMIT_EMBEDDED_B64 = "' + base64.b64encode( + module.CORE_SOURCE.with_name("saw_ui_limit.py").read_bytes()).decode("ascii") + '"\n' assert len(module.build_manifest(data).splitlines()) == len(BUNDLE_FILES) for name in BUNDLE_FILES[3:]: with pytest.raises(SystemExit, match=name): diff --git a/tests/installer/test_tool_gate.py b/tests/installer/test_tool_gate.py index fe9c94812..5e6c4c7f0 100644 --- a/tests/installer/test_tool_gate.py +++ b/tests/installer/test_tool_gate.py @@ -206,3 +206,23 @@ def test_before_tool_call_writes_hash_and_rule(tmp_path): assert line["policyHash"] == policy_hash and line["rule"] assert line["argsDigest"].startswith("sha256:") assert json.dumps(line, separators=(",", ":")) in result.stderr + + +@pytest.mark.ci_external +def test_missing_final_receipt_prevents_approved_effect(): + # Stock OpenClaw catches callback exceptions. The signed container path + # must stop the executor synchronously before control returns to the tool. + script = """ +import {evaluateToolCall} from "./tool-gate.mjs"; +import {readFileSync} from "node:fs"; +const held=evaluateToolCall({policyText:readFileSync("tool-actions.yaml","utf8"), + identity:{username:"alice",subject:"example",workspace:"w",sandbox:"s"}, + sessionKey:"test", toolName:"exec", params:{command:"rm fixture"}, + appendRecord(r){if(r.decision==="approved")throw Error("writer unavailable")}, + unacknowledgedApproval:()=>process.exit(75)}); +try {held.hook.requireApproval.onResolution("allow-once")} catch {} +console.log("TOOL_EXECUTED"); +""" + result = subprocess.run(["node","--input-type=module","-e",script], cwd=GATE, + capture_output=True,text=True) + assert result.returncode == 75 and "TOOL_EXECUTED" not in result.stdout diff --git a/tests/installer/test_ui_transport.py b/tests/installer/test_ui_transport.py new file mode 100644 index 000000000..8fa6c9603 --- /dev/null +++ b/tests/installer/test_ui_transport.py @@ -0,0 +1,121 @@ +"""Shared UI transport recovery without resending application requests.""" +import asyncio +import types + +import pytest + + +def transport(ab): + module = types.ModuleType("saw_ui_limit") + exec(compile(ab.SANDBOX_UI_LIMIT_PY, "saw_ui_limit.py", "exec"), module.__dict__) + return module + + +@pytest.mark.parametrize("method", [b"GET", b"POST"]) +def test_connect_only_queues_during_cli_listener_restart(ab, method): + module = transport(ab) + + async def scenario(): + seen = [] + + async def upstream(reader, writer): + seen.append(await reader.readuntil(b"\r\n\r\n")) + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + await writer.drain() + writer.close() + + reservation = await asyncio.start_server(upstream, "127.0.0.1", 0) + port = reservation.sockets[0].getsockname()[1] + reservation.close() + await reservation.wait_closed() + front = await asyncio.start_server(module.handler(port, module.Slots(16), retry_sent=False), "127.0.0.1", 0) + + async def restart(): + # Reproduce a token-expiry CLI restart: no listener for two seconds. + await asyncio.sleep(2.1) + return await asyncio.start_server(upstream, "127.0.0.1", port) + + restarting = asyncio.create_task(restart()) + try: + reader, writer = await asyncio.open_connection("127.0.0.1", front.sockets[0].getsockname()[1]) + request = method + b" / HTTP/1.1\r\nHost: fixture\r\nContent-Length: 0\r\n\r\n" + writer.write(request) + await writer.drain() + response = await asyncio.wait_for(reader.read(), 8) + writer.close() + assert response.startswith(b"HTTP/1.1 200") + assert seen == [request] + finally: + up = await restarting + up.close() + front.close() + await up.wait_closed() + await front.wait_closed() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("method", [b"GET", b"POST"]) +def test_connect_only_never_replays_after_any_request_was_sent(ab, method): + module = transport(ab) + + async def scenario(): + seen = [] + + async def upstream(reader, writer): + seen.append(await reader.readuntil(b"\r\n\r\n")) + writer.close() # An upstream may have processed it: never replay. + + up = await asyncio.start_server(upstream, "127.0.0.1", 0) + front = await asyncio.start_server(module.handler(up.sockets[0].getsockname()[1], module.Slots(16), retry_sent=False), "127.0.0.1", 0) + try: + reader, writer = await asyncio.open_connection("127.0.0.1", front.sockets[0].getsockname()[1]) + writer.write(method + b" / HTTP/1.1\r\nHost: fixture\r\n\r\n") + await writer.drain() + assert await asyncio.wait_for(reader.read(), 3) == b"" + writer.close() + assert len(seen) == 1 + finally: + up.close() + front.close() + await up.wait_closed() + await front.wait_closed() + + asyncio.run(scenario()) + + +def test_expired_websocket_is_closed_and_client_can_reconnect(ab): + module = transport(ab) + + async def scenario(): + seen = [] + expired = asyncio.Event() + + async def upstream(reader, writer): + seen.append(await reader.readuntil(b"\r\n\r\n")) + writer.write(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n\r\n") + await writer.drain() + if len(seen) == 1: + await expired.wait() + writer.close() + + up = await asyncio.start_server(upstream, "127.0.0.1", 0) + front = await asyncio.start_server(module.handler(up.sockets[0].getsockname()[1], module.Slots(16), retry_sent=False), "127.0.0.1", 0) + try: + for attempt in range(2): + reader, writer = await asyncio.open_connection("127.0.0.1", front.sockets[0].getsockname()[1]) + writer.write(b"GET /ws HTTP/1.1\r\nHost: fixture\r\nUpgrade: websocket\r\n\r\n") + await writer.drain() + assert b"101" in await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 3) + if attempt == 0: + expired.set() + assert await asyncio.wait_for(reader.read(), 3) == b"" + writer.close() + assert len(seen) == attempt + 1 + finally: + up.close() + front.close() + await up.wait_closed() + await front.wait_closed() + + asyncio.run(scenario()) diff --git a/tests/kubernetes_runtime/__init__.py b/tests/kubernetes_runtime/__init__.py new file mode 100644 index 000000000..93184981e --- /dev/null +++ b/tests/kubernetes_runtime/__init__.py @@ -0,0 +1 @@ +"""Keep runtime fixtures separate from the legacy installer's conftest module.""" diff --git a/tests/kubernetes_runtime/conftest.py b/tests/kubernetes_runtime/conftest.py new file mode 100644 index 000000000..7049a13d1 --- /dev/null +++ b/tests/kubernetes_runtime/conftest.py @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "kubernetes-runtime")) diff --git a/tests/kubernetes_runtime/profile_fixtures.py b/tests/kubernetes_runtime/profile_fixtures.py new file mode 100644 index 000000000..a41be17bd --- /dev/null +++ b/tests/kubernetes_runtime/profile_fixtures.py @@ -0,0 +1,64 @@ +"""Synthetic bindings for the actual Helm-rendered BOMs; no cluster workload.""" +import base64 +from pathlib import Path +import subprocess + +import yaml + +from saw_kubernetes.core import shared +from saw_kubernetes.harness import read_bundles + +ROOT = Path(__file__).resolve().parents[2] +IMAGE = "example.test/agent@sha256:" + "a" * 64 + + +def browser_slot(): + def pvc(name, read_only): + return {"claimName": name, "uid": "test-" + name, "readOnly": read_only} + + # Public SPKI for a fixture writer. No signing or provider key in the test. + key = base64.b64encode(bytes.fromhex("302a300506032b6570032100") + bytes(range(32))).decode() + return { + "workspace": pvc("workspace-fixture", False), + "browser": {"origin": "https://fixture.example", "ownerLogin": "fixture"}, + "toolGate": { + "receiptKey": key, + "ownerLogin": "fixture", + "trusted": pvc("saw-gate-fixture", True), + "audit": pvc("saw-audit-fixture", False), + }, + } + + +def briefing_inputs(image=IMAGE): + manifests = subprocess.check_output(["helm", "template", "fixture", str(ROOT/"charts/saw-bom"), + "--set", "profiles[0]=personal-assistant", "--set", "harnessEnabled=true", + "--set-string", "imageOverrides.assistant="+image], text=True) + files = yaml.safe_load(manifests)["data"] + docs, raw, index = shared.split_bom_files(files) + bundle = read_bundles(raw, index)["daily-briefing"] + slot = browser_slot() + slot["toolGate"]["harness"] = {"name": bundle.name, "digest": bundle.digest} + ws, = shared.parse_profiles(docs)[0].workspaces + bindings = [{"secret":p.credential_secret,"key":p.credential_secret_key,"type":p.type, + "refresh": bool(p.refresh_strategy)} for p in ws.providers if p.enabled] + cfg = dict(ownerID="b032b2e4-2cd5-47a7-8ef8-ac73e15700b1",namespace="saw-fixture", + namespaceUID="fixture-ns",ownerSubject="fixture-subject",mode="container",gatewayVersion="0.1.2-rhaiv.6", + approvedCredentialFreeProviders=[],proxyBindings=bindings,bindings={"personal-assistant/assistant":slot}) + return files, cfg + + +def fixture_inputs(): + site = yaml.safe_load((ROOT / "tests/fixtures/container-site.yaml").read_text()) + owner, = site["portal"]["containerSites"].values() + image = owner["profileImageOverrides"]["data-science"]["notebook"] + rendered = subprocess.check_output(["helm", "template", "fixture", str(ROOT/"charts/saw-bom"), + "--set", "profiles[0]=custom-inference", "--set", "webSearch.enabled=false", + "--set-string", "imageOverrides.notebook="+image], text=True) + cfg = dict(namespace="saw-fixture", namespaceUID="fixture", ownerSubject="fixture-owner", mode="container", + ownerID="b032b2e4-2cd5-47a7-8ef8-ac73e15700b1", gatewayVersion="0.1.2-rhaiv.6", + approvedCredentialFreeProviders=[], + proxyBindings=[{"type":"custom-inference","secret":"inference","key":"api_key","refresh":False}], + customInferenceURL="https://models.fixture.test/v1", modelInputs={"inference.model":"fixture/model"}, + bindings={"default/notebook":browser_slot()}) + return yaml.safe_load(rendered)["data"], cfg diff --git a/tests/kubernetes_runtime/test_admin_session.py b/tests/kubernetes_runtime/test_admin_session.py new file mode 100644 index 000000000..30f3a4d8e --- /dev/null +++ b/tests/kubernetes_runtime/test_admin_session.py @@ -0,0 +1,389 @@ +"""Per-operation identity, public trust updates and cleanup under failure.""" +import base64 +from contextlib import contextmanager +import copy +import json +import os +import ssl +import subprocess +import uuid + +import pytest + +from saw_kubernetes import admin_session as sessions +from saw_kubernetes.compiler import ContractError +from saw_kubernetes.controller import ApiError + +OWNER = "67a9891d-e281-442f-86c3-41771f5ce049" +PODS = {"controller": str(uuid.uuid4()), "bind-job": str(uuid.uuid4())} + + +@pytest.fixture +def ca(tmp_path): + subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", + "ec_paramgen_curve:prime256v1", "-nodes", "-keyout", str(tmp_path/"ca.key"), + "-out", str(tmp_path/"ca.crt"), "-days", "2", "-subj", "/CN=server-ca"], + check=True, capture_output=True) + return (tmp_path/"ca.crt").read_bytes() + + +def local_identity(ca, path): + # Unit fixture: exercise real issuance without requiring a Linux tmpfs or shred. + path.mkdir(exist_ok=True) + return sessions.admin.identity(ca, path, require_tmpfs=False, erase=lambda p: p.unlink()) + + +class Api: + def __init__(self, ca): + self.secret = {"apiVersion": "v1", "kind": "Secret", "type": "Opaque", "immutable": False, + "metadata": {"namespace": "runtime", "name": "gateway-client-ca", + "uid": str(uuid.uuid4()), "resourceVersion": "1", + "labels": {sessions.OWNER: OWNER}}, + "data": {"base.crt": sessions.encoded(ca), + "ca.crt": sessions.encoded(sessions.bundle(ca, {}))}} + self.pods = {name: {"metadata": {"uid": uid}, "status": {"phase": "Running"}} + for name, uid in PODS.items()} + self.writes, self.conflict, self.ambiguous = 0, None, False + + def request(self, method, path, body=None): + if path.startswith("/api/v1/namespaces/trusted/pods/"): + assert method == "GET" + name = path.rsplit("/", 1)[1] + if name not in self.pods: + raise ApiError(404) + return copy.deepcopy(self.pods[name]) + assert path == "/api/v1/namespaces/runtime/secrets/gateway-client-ca" + if method == "GET": + return copy.deepcopy(self.secret) + assert method == "PUT" + if self.conflict: + callback, self.conflict = self.conflict, None + callback() + if body["metadata"]["resourceVersion"] != self.secret["metadata"]["resourceVersion"]: + raise ApiError(409) + version = int(self.secret["metadata"]["resourceVersion"])+1 + self.secret = copy.deepcopy(body) + self.secret["metadata"]["resourceVersion"] = str(version) + self.writes += 1 + if self.ambiguous: + self.ambiguous = False + raise ApiError() + return copy.deepcopy(self.secret) + + def trust(self, ca, pod="controller", **kwargs): + return sessions.TrustBundle(self.request, "runtime", "trusted", "gateway", OWNER, + ca, pod, PODS[pod], **kwargs) + + +def published(api): + return json.loads(api.secret["metadata"].get("annotations", {}).get(sessions.SESSIONS, "{}")) + + +def test_shared_identity_roles_permissions_and_private_cleanup(ca, tmp_path): + runtime = tmp_path/"runtime" + with local_identity(ca, runtime) as identity: + tls = identity["tls"] + result = subprocess.run(["openssl", "x509", "-in", str(tls/"client/tls.crt"), + "-noout", "-text"], capture_output=True, text=True, check=True) + assert "openshell-admin" in result.stdout + assert "TLS Web Client Authentication" in result.stdout and "CA:FALSE" in result.stdout + assert (tls/"ca.crt").read_bytes() == ca + assert not list(runtime.rglob("ca.key")) + assert os.stat(tls/"client/tls.key").st_mode & 0o777 == 0o600 + assert os.stat(tls.parent).st_mode & 0o777 == 0o700 + assert not tls.exists() and not list(runtime.rglob("*.key")) + + +def test_starting_job_waits_for_its_own_running_phase_before_issuing_or_publishing(ca, tmp_path, monkeypatch): + api = Api(ca) + runtime = tmp_path/"runtime" + runtime.mkdir() + api.pods["bind-job"]["status"]["phase"] = "Pending" + trust = api.trust(ca, "bind-job") + elapsed = [0] + def sleep(seconds): + assert not published(api) and api.writes == 0 + assert not list(runtime.rglob("*.key")) and not list(runtime.glob("saw-admin.*")) + elapsed[0] += seconds + if elapsed[0] >= 1: + api.pods["bind-job"]["status"]["phase"] = "Running" + wait = trust.wait_for_holder + monkeypatch.setattr(trust, "wait_for_holder", lambda: wait(sleep=sleep, monotonic=lambda: elapsed[0])) + identity = sessions.admin.identity + subjects = [] + def issue_identity(ca, runtime, **kwargs): + subjects.append(kwargs.get("subject")) + return identity(ca, runtime, require_tmpfs=False, erase=lambda p: p.unlink(), **kwargs) + monkeypatch.setattr(sessions.admin, "identity", issue_identity) + monkeypatch.setattr(sessions.admin, "require_memory", lambda _: None) + calls = [] + class Gateway: + def __init__(self, *args, **kwargs): pass + def call(self, method, body): + assert published(api) and trust.live({"pod": trust.pod, "uid": trust.pod_uid}) + calls.append(method) + def close(self): pass + with sessions.GatewaySessions(trust, "https://gateway:17670", runtime, gateway_factory=Gateway).open(): + assert elapsed[0] == 1 and calls == ["GetGatewayInfo"] + assert subjects == [sessions.OPERATOR_SUBJECT] + assert len(published(api)) == 1 + assert not published(api) and not list(runtime.rglob("*.key")) + + +@pytest.mark.parametrize("state", ["missing", "replacement", "deleting", "Succeeded", "Failed", "Unknown", "unset"]) +def test_startup_wait_fails_closed_for_nonstarting_identity(ca, tmp_path, monkeypatch, state): + api = Api(ca) + pod = api.pods["bind-job"] + if state == "missing": del api.pods["bind-job"] + elif state == "replacement": pod["metadata"]["uid"] = str(uuid.uuid4()) + elif state == "deleting": pod["metadata"]["deletionTimestamp"] = "now" + elif state == "unset": pod["status"] = {} + else: pod["status"]["phase"] = state + monkeypatch.setattr(sessions.admin, "require_memory", lambda _: None) + def identity(*args, **kwargs): pytest.fail("issued private identity before holder readiness") + monkeypatch.setattr(sessions.admin, "identity", identity) + with pytest.raises(ContractError, match="administrator session holder"): + with sessions.GatewaySessions(api.trust(ca, "bind-job"), "https://gateway:17670", tmp_path).open(): + pytest.fail("unavailable holder was admitted") + assert not published(api) and api.writes == 0 + + +def test_startup_wait_timeout_is_bounded_and_does_not_publish(ca): + api = Api(ca) + api.pods["bind-job"]["status"]["phase"] = "Pending" + elapsed = [0] + with pytest.raises(ContractError, match="before the deadline"): + api.trust(ca, "bind-job").wait_for_holder(timeout=2, + sleep=lambda seconds: elapsed.__setitem__(0, elapsed[0]+seconds), monotonic=lambda: elapsed[0]) + assert elapsed[0] == 2 and api.writes == 0 and not published(api) + + +@pytest.mark.parametrize("code", [0, 403, 503]) +def test_startup_wait_does_not_retry_api_errors_or_publish(ca, code): + api = Api(ca) + trust = api.trust(ca, "bind-job") + def unavailable(*args, **kwargs): raise ApiError(code) + trust.request = unavailable + def no_sleep(_): pytest.fail("retried API error as a Pending Pod") + with pytest.raises(ApiError) as error: + trust.wait_for_holder(sleep=no_sleep) + assert error.value.code == code and api.writes == 0 + + +def test_publication_rechecks_uid_after_successful_startup_wait(ca, tmp_path, monkeypatch): + api = Api(ca) + runtime = tmp_path/"runtime" + runtime.mkdir() + monkeypatch.setattr(sessions.admin, "require_memory", lambda _: None) + original = sessions.admin.identity + @contextmanager + def identity(base, runtime, **kwargs): + with original(base, runtime, require_tmpfs=False, erase=lambda p: p.unlink(), **kwargs) as value: + api.pods["bind-job"]["metadata"]["uid"] = str(uuid.uuid4()) + yield value + monkeypatch.setattr(sessions.admin, "identity", identity) + with pytest.raises(ContractError, match="holder unavailable"): + with sessions.GatewaySessions(api.trust(ca, "bind-job"), "https://gateway:17670", runtime).open(): + pytest.fail("replacement holder was admitted") + assert not published(api) and api.writes == 0 and not list(runtime.rglob("*.key")) + + +def test_concurrent_sessions_preserve_trust_with_compare_and_swap(ca, tmp_path): + api = Api(ca) + first, second = api.trust(ca), api.trust(ca, "bind-job") + with local_identity(ca, tmp_path/"one") as one, local_identity(ca, tmp_path/"two") as two: + api.conflict = lambda: second.update(add=two) + first.update(add=one) + assert set(published(api)) == {one["id"], two["id"]} + assert all(b"PRIVATE KEY" not in base64.b64decode(v) for v in api.secret["data"].values()) + first.update(remove=one["id"]) + assert set(published(api)) == {two["id"]} + second.update(remove=two["id"]) + assert not published(api) + assert api.secret["data"]["ca.crt"] == sessions.encoded(sessions.bundle(ca, {})) + + +@pytest.mark.parametrize("dead", ["deleted", "replaced", "terminating", "completed", "expired"]) +def test_abandoned_session_pruned_without_dropping_live_session(ca, tmp_path, dead): + api = Api(ca) + with local_identity(ca, tmp_path/"one") as one, local_identity(ca, tmp_path/"two") as two: + api.trust(ca).update(add=one) + api.trust(ca, "bind-job").update(add=two) + if dead == "deleted": del api.pods["controller"] + if dead == "replaced": api.pods["controller"]["metadata"]["uid"] = str(uuid.uuid4()) + if dead == "terminating": api.pods["controller"]["metadata"]["deletionTimestamp"] = "now" + if dead == "completed": api.pods["controller"]["status"]["phase"] = "Succeeded" + if dead == "expired": + values = published(api) + values[one["id"]]["expires"] = 1 + api.secret["metadata"]["annotations"][sessions.SESSIONS] = json.dumps(values) + api.trust(ca, "bind-job").update(add=two) + assert set(published(api)) == {two["id"]} + + +@pytest.mark.parametrize("damage", ["owner", "immutable", "base", "bundle", "private", "uid", "gc"]) +def test_foreign_or_malformed_trust_is_not_written(ca, tmp_path, damage): + api = Api(ca) + trust = api.trust(ca) + with local_identity(ca, tmp_path/"one") as one: + trust.update(add=one) + if damage == "owner": api.secret["metadata"]["labels"][sessions.OWNER] = "foreign" + if damage == "immutable": api.secret["immutable"] = True + if damage == "base": api.secret["data"]["base.crt"] = sessions.encoded(b"foreign") + if damage == "bundle": api.secret["data"]["ca.crt"] = sessions.encoded(ca) + if damage == "uid": api.secret["metadata"]["uid"] = str(uuid.uuid4()) + if damage == "gc": api.secret["metadata"]["ownerReferences"] = [{"uid": "foreign"}] + if damage == "private": + value = one["ca"].encode()+(one["tls"]/"client/tls.key").read_bytes() + api.secret["data"]["session-"+one["id"]+".crt"] = sessions.encoded(value) + api.secret["data"]["ca.crt"] = sessions.encoded(sessions.bundle(ca, {one["id"]: value})) + before = api.writes + with pytest.raises(ContractError): trust.update(remove=one["id"]) + assert api.writes == before + + +def test_no_withdrawal_of_another_live_holder(ca, tmp_path): + api = Api(ca) + with local_identity(ca, tmp_path/"one") as one: + api.trust(ca).update(add=one) + with pytest.raises(ContractError): api.trust(ca, "bind-job").update(remove=one["id"]) + assert one["id"] in published(api) + + +def test_pod_api_error_does_not_prune_trust(ca, tmp_path): + api = Api(ca) + trust = api.trust(ca) + with local_identity(ca, tmp_path/"one") as one: + trust.update(add=one) + def unavailable(holder): raise ApiError(503) + trust.live = unavailable + with pytest.raises(ApiError): trust.update(remove=one["id"]) + assert one["id"] in published(api) + + +@pytest.mark.parametrize("failure", ["operation", "ambiguous-publish", "probe"]) +def test_session_cleanup_on_every_failure(ca, tmp_path, monkeypatch, failure): + api = Api(ca) + identity_factory = sessions.admin.identity + def identity(base, runtime, **kwargs): + return identity_factory(base, runtime, require_tmpfs=False, erase=lambda p: p.unlink(), **kwargs) + monkeypatch.setattr(sessions.admin, "identity", identity) + monkeypatch.setattr(sessions.admin, "require_memory", lambda _: None) + clients = [] + class Gateway: + def __init__(self, *args, **kwargs): + self.closed = False + clients.append(self) + def call(self, method, body): + assert method == "GetGatewayInfo" + if failure == "probe": raise sessions.RpcError("PERMISSION_DENIED") + def close(self): self.closed = True + api.ambiguous = failure == "ambiguous-publish" + factory = sessions.GatewaySessions(api.trust(ca), "https://gateway:17670", tmp_path, gateway_factory=Gateway) + with pytest.raises((ApiError, RuntimeError, sessions.RpcError)): + with factory.open(): + assert published(api) + raise RuntimeError("operation failed") + assert not published(api) + assert all(client.closed for client in clients) + assert not list(tmp_path.rglob("tls.key")) + + +def test_trust_wait_is_bounded(): + elapsed = [0] + def sleep(seconds): elapsed[0] += seconds + with pytest.raises(sessions.admin.AdminSessionError): + sessions.admin.wait_for_trust(lambda: False, timeout=3, sleep=sleep, monotonic=lambda: elapsed[0]) + assert elapsed[0] == 3 + + +def test_temporary_identity_authenticates_only_while_its_ca_is_trusted(ca, tmp_path): + """Real TLS over memory BIOs; no sockets or gateway-specific reload simulation.""" + def handshake(identity, ca_bundle): + server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server.maximum_version = ssl.TLSVersion.TLSv1_2 + server.load_cert_chain(tmp_path/"ca.crt", tmp_path/"ca.key") + server.verify_mode = ssl.CERT_REQUIRED + server.load_verify_locations(cadata=ca_bundle.decode()) + client = ssl.create_default_context(cadata=ca.decode()) + client.load_cert_chain(identity["tls"]/"client/tls.crt", identity["tls"]/"client/tls.key") + si, so, ci, co = (ssl.MemoryBIO() for _ in range(4)) + endpoints = [(client.wrap_bio(ci, co, server_hostname="server-ca"), co, si), + (server.wrap_bio(si, so, server_side=True), so, ci)] + done = set() + for _ in range(20): + for index, (connection, outgoing, incoming) in enumerate(endpoints): + try: + connection.do_handshake() + done.add(index) + except ssl.SSLWantReadError: + pass + while outgoing.pending: + incoming.write(outgoing.read()) + if len(done) == 2: return + pytest.fail("TLS handshake did not complete") + + api = Api(ca) + trust = api.trust(ca) + with local_identity(ca, tmp_path/"one") as one: + with pytest.raises(ssl.SSLError): handshake(one, ca) + trust.update(add=one) + handshake(one, base64.b64decode(api.secret["data"]["ca.crt"])) + trust.update(remove=one["id"]) + with pytest.raises(ssl.SSLError): handshake(one, base64.b64decode(api.secret["data"]["ca.crt"])) + with local_identity(ca, tmp_path/"two") as two: + assert one["id"] != two["id"] and one["ca"] != two["ca"] + trust.update(add=two) + handshake(two, base64.b64decode(api.secret["data"]["ca.crt"])) + with pytest.raises(ssl.SSLError): handshake(one, base64.b64decode(api.secret["data"]["ca.crt"])) + trust.update(remove=two["id"]) + + +def test_same_pod_restart_removes_its_abandoned_session_only(ca, tmp_path, monkeypatch): + api = Api(ca) + runtime = tmp_path/"runtime" + runtime.mkdir() + stale = runtime/"saw-admin.abandoned" + stale.mkdir() + (stale/"tls.key").write_text("private fixture") + monkeypatch.setattr(sessions.admin, "require_memory", lambda _: None) + with local_identity(ca, tmp_path/"old") as old, local_identity(ca, tmp_path/"relay") as relay: + api.trust(ca).update(add=old) + api.trust(ca, "bind-job").update(add=relay) + factory = sessions.GatewaySessions(api.trust(ca), "https://gateway:17670", runtime) + with factory._exclusive(): + assert not stale.exists() + assert set(published(api)) == {relay["id"]} + # A second process must not remove the current process's files/trust. + fresh = runtime/"saw-admin.active" + fresh.mkdir() + other = sessions.GatewaySessions(api.trust(ca), "https://gateway:17670", runtime) + with pytest.raises(ContractError, match="another administrator operation"): + with other._exclusive(): pytest.fail("concurrent session was admitted") + assert fresh.exists() and set(published(api)) == {relay["id"]} + + +def test_restart_erases_private_files_even_when_trust_api_is_unavailable(ca, tmp_path, monkeypatch): + stale = tmp_path/"saw-admin.abandoned" + stale.mkdir() + (stale/"tls.key").write_text("private fixture") + api = Api(ca) + def unavailable(*args, **kwargs): raise ApiError(503) + trust = api.trust(ca) + trust.request = unavailable + monkeypatch.setattr(sessions.admin, "require_memory", lambda _: None) + with pytest.raises(ApiError): + with sessions.GatewaySessions(trust, "https://gateway:17670", tmp_path)._exclusive(): + pytest.fail("operation proceeded without trust recovery") + assert not stale.exists() + + +def test_abandoned_cleanup_does_not_follow_authority_symlink(tmp_path): + root, outside = tmp_path/"runtime", tmp_path/"outside" + root.mkdir() + outside.mkdir() + (outside/"session.keep").mkdir() + (root/"saw-authorities").symlink_to(outside) + with pytest.raises(sessions.admin.AdminSessionError, match="symlink"): + sessions.admin.remove_abandoned(root) + assert (outside/"session.keep").exists() diff --git a/tests/kubernetes_runtime/test_boundary.py b/tests/kubernetes_runtime/test_boundary.py new file mode 100644 index 000000000..ca29e3bd0 --- /dev/null +++ b/tests/kubernetes_runtime/test_boundary.py @@ -0,0 +1,90 @@ +import pytest + +from saw_kubernetes.audit import capture +from saw_kubernetes.boundary import BoundaryError, objects, verify + + +@pytest.fixture +def installation(): + config = dict(namespace="runtime", namespaceUID="ns-uid", ownerID="owner", + mode="container", trustedNamespace="trusted", governanceNamespace="gov", + gateway="gateway", proxyPolicy="proxy") + resources = {} + for key, (path, field) in objects(config).items(): + resources[path] = {"metadata": {"uid": key, "generation": 1}, field: {}} + def obj(key): + return resources[objects(config)[key][0]] + for key in ("validation", "secretGuard", "serviceGuard", "podDeleteGuard"): + obj(key)["status"] = {"observedGeneration": 1, "typeChecking": {}} + for key in ("gateway", "governance", "proxy"): + obj(key)["spec"]["replicas"] = 1 + obj(key)["status"] = {"observedGeneration": 1, "readyReplicas": 1, "updatedReplicas": 1} + obj("network")["spec"]["priority"] = 50 + obj("network")["status"] = {"conditions": [ + {"type": "Ready-In-Zone-worker", "status": "True", "reason": "SetupSucceeded"}]} + resources["/api/v1/nodes"] = {"items": [{"metadata": {"name": "worker"}}]} + listing = "/apis/policy.networking.k8s.io/v1alpha1/adminnetworkpolicies" + resources[listing] = {"items": [obj("network")]} + approved = capture(config, resources.__getitem__) + return approved, resources, obj, listing + + +@pytest.mark.parametrize("case,reason", [ + ("replaced", "ChangedBoundary:proxy"), ("drift", "ChangedBoundary:publicCA"), + ("new-zone", "NetworkNotRealized"), ("precedence", "EarlierNetworkPolicy"), + ("warning", "AdmissionUnavailable"), ("rollout", "RolloutIncomplete:gateway"), +]) +def test_changed_or_unrealized_boundary_blocks(installation, case, reason): + config, resources, obj, listing = installation + if case == "replaced": + obj("proxy")["metadata"]["uid"] = "replacement" + elif case == "drift": + obj("publicCA")["data"]["ca.crt"] = "new" + elif case == "new-zone": + resources["/api/v1/nodes"]["items"].append({"metadata": {"name": "new"}}) + elif case == "precedence": + resources[listing]["items"].append({"metadata": {"uid": "other"}, "spec": {"priority": 50}}) + elif case == "warning": + obj("validation")["status"]["typeChecking"]["expressionWarnings"] = [{"warning": "invalid"}] + elif case == "rollout": + obj("gateway")["status"]["updatedReplicas"] = 0 + with pytest.raises(BoundaryError, match=reason): + verify(config, config, resources.__getitem__) + + +@pytest.mark.parametrize("subject,disjoint", [ + ({"pods": {"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "other"}}, + "podSelector": {}}}, True), + ({"namespaces": {"matchLabels": {"kubernetes.io/metadata.name": "other"}}}, True), + ({"pods": {"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "runtime"}}}}, False), + ({"namespaces": {"matchLabels": {"owner": "other"}}}, False), + ({"pods": {"podSelector": {"matchLabels": {"kubernetes.io/metadata.name": "other"}}}}, False), + ({"namespaces": {}, "pods": {}}, False), +]) +def test_second_tenant_network_precedence(installation, subject, disjoint): + config, resources, _, listing = installation + resources[listing]["items"].append({"metadata": {"uid": "first"}, + "spec": {"priority": 10, "subject": subject}}) + if disjoint: + verify(config, config, resources.__getitem__) + else: + with pytest.raises(BoundaryError, match="EarlierNetworkPolicy"): + verify(config, config, resources.__getitem__) + + +def test_kata_review_includes_sandbox_admission_and_gateway_placement(installation): + config, resources, _, _ = installation + config = dict(config, mode="kata", workloadRuntimeClass="saw-kata") + refs = objects(config) + for key in ("sandboxPlacement", "sandboxPlacementBinding"): + resources[refs[key][0]] = {"metadata": {"uid": key, "generation": 1}, "spec": {}} + resources[refs["sandboxPlacement"][0]]["status"] = {"observedGeneration": 1, "typeChecking": {}} + gateway = resources[refs["gatewayConfig"][0]]["data"] + gateway["gateway.toml"] = '[openshell.drivers.kubernetes]\ndefault_runtime_class_name = "saw-kata"' + approved = capture(config, resources.__getitem__) + assert set(approved["approved"]) >= {"sandboxPlacement", "sandboxPlacementBinding"} + with pytest.raises(BoundaryError, match="WrongWorkloadPlacement"): + verify(approved, dict(config, runtimeClass={"name": "other"}), resources.__getitem__) + gateway["gateway.toml"] = '[openshell.drivers.kubernetes]\ndefault_runtime_class_name = ""' + with pytest.raises(BoundaryError, match="WrongWorkloadPlacement"): + capture(config, resources.__getitem__) diff --git a/tests/kubernetes_runtime/test_briefing.py b/tests/kubernetes_runtime/test_briefing.py new file mode 100644 index 000000000..3c4a2947a --- /dev/null +++ b/tests/kubernetes_runtime/test_briefing.py @@ -0,0 +1,47 @@ +"""Compile the same Helm-rendered personal-assistant BOM as the VM.""" +import base64 +from pathlib import Path + +import pytest + +from saw_kubernetes import compile_profile_files, ContractError +from saw_kubernetes.core import shared +from saw_kubernetes.harness import read_bundles +from saw_kubernetes.prepare_gate import publish_harness + +from .profile_fixtures import ROOT, briefing_inputs + + +def test_briefing_plan_and_volume(tmp_path): + files, cfg = briefing_inputs() + plan = compile_profile_files(files, cfg) + ws, = plan["workspaces"] + sb, = ws["sandboxes"] + spec = sb["spec"] + assert {p["type"]:p["credentials"] for p in ws["providers"]} == { + "gmail":{"GMAIL_ACCESS_TOKEN":"saw-cred.gmail.access_token"}, + "slack":{"SLACK_BOT_TOKEN":"saw-cred.slack.bot_token"}, + "nvidia":{"NVIDIA_API_KEY":"saw-cred.inference.api_key"}} + mount, = [m for m in spec["template"]["driver_config"]["kubernetes"]["containers"]["agent"]["volume_mounts"] + if m["mount_path"]=="/sandbox/harness"] + assert mount["read_only"] and mount["name"]=="saw-gate" + bom=tmp_path/"bom"; gate=tmp_path/"gate"; bom.mkdir(); gate.mkdir() + for k,v in files.items(): (bom/k).write_text(v) + pin="sha256:"+mount["sub_path"].removeprefix("harness-") + for _ in range(2): publish_harness(gate,bom,"daily-briefing",pin) + path=gate/mount["sub_path"]/"mcp/slack-reader.mjs" + assert path.read_bytes()==(ROOT/"harness-bundles/daily-briefing/mcp/slack-reader.mjs").read_bytes() + path.chmod(0o600); path.write_text("tampered") + with pytest.raises(ValueError): publish_harness(gate,bom,"daily-briefing",pin) + + +@pytest.mark.parametrize("fault", ["missing","edited","no-approval","missing-provider","no-index"]) +def test_untrusted_briefing_is_rejected(fault): + files,cfg=briefing_inputs() + ks=sorted(k for k in files if k.startswith("harness__") and not k.endswith("__map")) + if fault=="missing": del files[ks[0]] + if fault=="edited": files[ks[0]]=base64.b64encode(b"tampered").decode() + if fault=="no-index": del files["harness-index.yaml"] + if fault=="no-approval": del cfg["bindings"]["personal-assistant/assistant"]["toolGate"]["harness"] + if fault=="missing-provider": cfg["proxyBindings"]=[b for b in cfg["proxyBindings"] if b["type"]!="gmail"] + with pytest.raises(ContractError): compile_profile_files(files,cfg) diff --git a/tests/kubernetes_runtime/test_controller.py b/tests/kubernetes_runtime/test_controller.py new file mode 100644 index 000000000..d48657961 --- /dev/null +++ b/tests/kubernetes_runtime/test_controller.py @@ -0,0 +1,509 @@ +"""Named-ConfigMap controller contracts with no cluster/network or real credentials.""" + +import copy +import io +import json +from urllib.error import HTTPError + +import pytest + +from saw_kubernetes.compiler import ContractError, OWNER, canonical, digest +from saw_kubernetes.controller import ( + ApiError, CONTROLLER, Controller, ConfigMapClient, MANAGED, MAX_BODY, + Scope, _NoRedirect, +) +from saw_kubernetes.reconcile import Busy, RpcError +from .test_runtime import FakeGateway, flat_profile, source + + +OWNER_ID = "b032b2e4-2cd5-47a7-8ef8-ac73e15700b1" + + +class FakeAPI: + def __init__(self, scope): + self.scope = scope + self.objects, self.calls, self.writes, self.watches = {}, [], [], [] + self.version = 10 + self.conflicts = 0 + self.on_get = None + self.on_watch = lambda: iter(()) + + def configmap(self, name, data, **metadata): + self.version += 1 + obj = {"apiVersion": "v1", "kind": "ConfigMap", "metadata": { + "name": name, "namespace": self.scope.namespace, "uid": name + "-uid", + "resourceVersion": str(self.version), **metadata}, "data": copy.deepcopy(data)} + self.objects[name] = obj + return obj + + def get(self, name): + assert name in {self.scope.profile_name, self.scope.bindings_name, self.scope.status_name} + self.calls.append(name) + if self.on_get: + self.on_get(name) + return copy.deepcopy(self.objects.get(name)) + + def write_status(self, body, *, resource_version=None): + assert body["metadata"]["name"] == self.scope.status_name + self.writes.append((copy.deepcopy(body), resource_version)) + if self.conflicts: + self.conflicts -= 1 + raise ApiError(409) + old = self.objects.get(self.scope.status_name) + if (old is not None and resource_version != old["metadata"]["resourceVersion"] + or old is None and resource_version is not None): + raise ApiError(409) + self.version += 1 + saved = copy.deepcopy(body) + saved["metadata"].update(uid=self.scope.status_name + "-uid", resourceVersion=str(self.version)) + self.objects[self.scope.status_name] = saved + return copy.deepcopy(saved) + + def watch(self, name, resource_version, *, timeout_seconds): + self.watches.append((name, resource_version, timeout_seconds)) + return self.on_watch() + + +@pytest.fixture +def setup(tmp_path): + scope = Scope("saw-alice", "saw-alice-bom", "saw-runtime-intent", "saw-runtime-status", OWNER_ID) + api = FakeAPI(scope) + files = {"profiles__generic__default__workspace.yaml": "workspace source", + "profiles__generic__default__sandbox.yaml": "sandbox source"} + profile = api.configmap(scope.profile_name, files) + intent = {"schemaVersion": 1, "generation": 1, "desiredState": "Running", + "profileRef": {"name": scope.profile_name, "uid": profile["metadata"]["uid"], "sha256": digest(files)}, + "bindings": {"ownerID": OWNER_ID, "namespace": scope.namespace}} + api.configmap(scope.bindings_name, {"intent.json": canonical(intent)}, labels={OWNER: OWNER_ID}) + compiled, calls, verified = [], [], [] + + def compile_files(flat, bindings): + compiled.append((copy.deepcopy(flat), copy.deepcopy(bindings))) + return {"flat": flat, "bindings": bindings} + + def core(plan, **kwargs): + calls.append((copy.deepcopy(plan), kwargs)) + return {"phase": "GenericReady", "retry_after_seconds": 0, + "message": "must-not-be-published", "credentials": "must-not-be-published"} + + controller = Controller(api, scope, tmp_path / "state", compile_files, core, lambda plan: verified.append(plan)) + return controller, api, intent, compiled, calls, verified + + +def update_intent(setup, change): + controller, api, intent, *_ = setup + change(intent) + api.version += 1 + obj = api.objects[controller.scope.bindings_name] + obj["data"] = {"intent.json": canonical(intent)} + obj["metadata"]["resourceVersion"] = str(api.version) + + +def test_existing_flat_files_compile_before_verified_core_pass(setup): + c, api, intent, compiled, calls, verified = setup + status = c.reconcile_once() + assert status["phase"] == "GenericReady" and status["observedGeneration"] == 1 + assert calls[0][1] == {"generation": 1, "desired_state": "Running"} + assert compiled[0][0] == api.objects[c.scope.profile_name]["data"] + assert verified == [calls[0][0]] + assert "must-not-be-published" not in canonical(api.objects[c.scope.status_name]) + before = len(api.writes) + c.reconcile_once() + assert len(api.writes) == before, "unchanged status must not churn resourceVersions" + state = c.store.load() + assert state["inputUIDs"] == {"bindings": c.scope.bindings_name + "-uid", "profile": c.scope.profile_name + "-uid"} + assert state["intent"]["revision"] == status["inputRevision"] + + +def test_real_shared_compiler_and_reconciler_compose_with_restart_and_stop(setup, source): + from saw_kubernetes import Reconciler, compile_profile_files + + c, api, _, _, _, _ = setup + files = flat_profile(source) + api.objects[c.scope.profile_name]["data"] = files + update_intent(setup, lambda i: i.update(bindings=source[1], + profileRef={**i["profileRef"], "sha256": digest(files)})) + gateway = FakeGateway() + state_root = c.store.directory.parent + reconciler = Reconciler(gateway, state_root, lambda plan: None) + c.compile_files, c.core_pass = compile_profile_files, reconciler.reconcile + assert c.reconcile_once()["phase"] == "Pending" # owner membership + assert c.reconcile_once()["phase"] == "Pending" # Sandbox created + instance = gateway.sandboxes[("default", "toolbox")] + instance["status"]["phase"] = "SANDBOX_PHASE_READY" + sandbox_id = instance["metadata"]["id"] + resumed = Controller(api, c.scope, state_root, compile_profile_files, + Reconciler(gateway, state_root, lambda plan: None).reconcile, lambda plan: None) + assert resumed.reconcile_once()["phase"] == "GenericReady" + assert sum(method == "CreateSandbox" for method, _ in gateway.calls) == 1 + resumed.verify_boundary = lambda _: (_ for _ in ()).throw(RuntimeError("revoked")) + assert resumed.reconcile_once()["reason"] == "BoundaryRejected" + update_intent(setup, lambda i: i.update(generation=2, desiredState="Stopped")) + assert resumed.reconcile_once()["phase"] == "Pending" + instance["status"]["phase"] = "SANDBOX_PHASE_STOPPED" + assert resumed.reconcile_once()["phase"] == "Stopped" + assert instance["metadata"]["id"] == sandbox_id + assert sum(method == "CreateSandbox" for method, _ in gateway.calls) == 1 + + +def test_gateway_contract_rejection_reports_a_fixed_reason(setup, source): + from saw_kubernetes import Reconciler, compile_profile_files + + c, api, _, _, _, _ = setup + files = flat_profile(source) + api.objects[c.scope.profile_name]["data"] = files + update_intent(setup, lambda i: i.update(bindings=source[1], + profileRef={**i["profileRef"], "sha256": digest(files)})) + gateway = FakeGateway() + original_call = gateway.call + + def wrong_gateway_version(method, request): + if method == "GetGatewayInfo": + return {"gateway_version": "unexpected", "compute_drivers": [{"name": "kubernetes"}]} + return original_call(method, request) + + gateway.call = wrong_gateway_version + reconciler = Reconciler(gateway, c.store.directory.parent, lambda _: None) + c.compile_files, c.core_pass = compile_profile_files, reconciler.reconcile + status = c.reconcile_once() + saved = canonical(api.objects[c.scope.status_name]) + assert status["reason"] == "GatewayMismatch" + assert status["phase"] == "Blocked" + assert "unexpected" not in saved + + +def test_missing_boundary_verifier_fails_before_api_access(setup): + c, api, *_ = setup + with pytest.raises(ContractError, match="boundary verifier"): + Controller(api, c.scope, c.store.directory, c.compile_files, c.core_pass, None) + assert api.calls == [] + + +@pytest.mark.parametrize("bad_name", ["a.-b", "a-.b", "a..b", "a.-.b"]) +def test_scope_requires_each_dns_subdomain_label_to_be_valid(bad_name): + with pytest.raises(ContractError, match="explicit Kubernetes names"): + Scope("saw-alice", bad_name, "intent", "status", OWNER_ID) + + +def test_scope_accepts_valid_dotted_configmap_names(): + scope = Scope("saw-alice", "profile.prod-1", "intent.prod-1", "status.prod-1", OWNER_ID) + assert scope.profile_name == "profile.prod-1" + + +@pytest.mark.parametrize("missing", ["profile_name", "bindings_name"]) +def test_missing_or_deleted_input_never_implies_stop_or_purge(setup, missing): + c, api, _, _, calls, _ = setup + c.reconcile_once() + del api.objects[getattr(c.scope, missing)] + assert c.reconcile_once()["reason"] == "MissingInputs" + assert len(calls) == 1 + assert {body["metadata"]["name"] for body, _ in api.writes} == {c.scope.status_name} + + +def test_gitops_half_revision_waits_without_mutating_then_converges(setup): + c, api, _, _, calls, _ = setup + c.reconcile_once() + profile = api.objects[c.scope.profile_name] + profile["data"]["profiles__generic__default__sandbox.yaml"] = "new source" + profile["metadata"]["resourceVersion"] = "profile-new" + assert c.reconcile_once()["reason"] == "InputRevisionMismatch" + assert len(calls) == 1 + update_intent(setup, lambda i: i.update(generation=2, profileRef={**i["profileRef"], "sha256": digest(profile["data"])})) + assert c.reconcile_once()["phase"] == "GenericReady" + assert calls[-1][1]["generation"] == 2 + + +def test_change_during_double_read_refuses_mixed_snapshot(setup): + c, api, _, _, calls, _ = setup + reads = 0 + + def change(name): + nonlocal reads + if name == c.scope.profile_name: + reads += 1 + if reads == 2: + api.objects[name]["metadata"]["resourceVersion"] = "changed" + + api.on_get = change + assert c.reconcile_once()["reason"] == "InputsChangedDuringRead" + assert not calls + + +@pytest.mark.parametrize("source_name", ["profile_name", "bindings_name"]) +def test_restart_refuses_same_name_input_uid_replacement(setup, source_name): + c, api, _, _, calls, _ = setup + c.reconcile_once() + name = getattr(c.scope, source_name) + api.objects[name]["metadata"]["uid"] = "replacement" + if source_name == "profile_name": + update_intent(setup, lambda i: i["profileRef"].update(uid="replacement")) + restarted = Controller(api, c.scope, c.store.directory.parent, c.compile_files, c.core_pass, c.verify_boundary) + assert restarted.reconcile_once()["reason"] == "InputIdentityReplaced" + assert len(calls) == 1 + + +def test_generation_cannot_roll_back_or_change_same_intent(setup): + c, _, _, _, calls, _ = setup + c.reconcile_once() + update_intent(setup, lambda i: i.update(desiredState="Stopped")) + assert c.reconcile_once()["reason"] == "StaleOrConflictingGeneration" + assert len(calls) == 1 + update_intent(setup, lambda i: i.update(generation=2)) + c.reconcile_once() + assert calls[-1][1] == {"generation": 2, "desired_state": "Stopped"} + update_intent(setup, lambda i: i.update(generation=1)) + assert c.reconcile_once()["reason"] == "StaleOrConflictingGeneration" + + +def test_boundary_failure_is_sanitized_and_prevents_core(setup): + c, api, _, _, calls, _ = setup + c.verify_boundary = lambda _: (_ for _ in ()).throw(RuntimeError("secret-token-and-URL")) + status = c.reconcile_once() + assert status["reason"] == "BoundaryRejected" + assert not calls and "secret-token" not in canonical(api.objects[c.scope.status_name]) + + +def test_ambiguous_core_failure_retains_intent_and_does_not_publish_exception(setup): + c, api, *_ = setup + c.core_pass = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("private-request-body")) + assert c.reconcile_once()["reason"] == "CoreFailure" + assert c.store.load()["intent"]["generation"] == 1 + assert "private-request-body" not in canonical(api.objects[c.scope.status_name]) + + +@pytest.mark.parametrize("safe_reason,expected", [ + ("GatewayMismatch", "GatewayMismatch"), + ("private-request-body", "CoreRejected"), +]) +def test_core_contract_rejection_publishes_only_allowlisted_reason(setup, safe_reason, expected): + c, api, *_ = setup + + def reject(*args, **kwargs): + raise ContractError("private-request-body", safe_reason=safe_reason) + + c.core_pass = reject + status = c.reconcile_once() + serialized = canonical(api.objects[c.scope.status_name]) + assert status["reason"] == expected + assert "private-request-body" not in serialized + + +def test_core_rpc_rejection_publishes_only_allowlisted_code(setup): + c, api, *_ = setup + c.core_pass = lambda *a, **k: (_ for _ in ()).throw(RpcError("FAILED_PRECONDITION")) + status = c.reconcile_once() + serialized = canonical(api.objects[c.scope.status_name]) + assert status["reason"] == "CoreRpcRejected" + assert status["rpcCode"] == "FAILED_PRECONDITION" + assert "FAILED_PRECONDITION" in serialized + assert "private" not in serialized + + +def test_profile_contract_failure_has_a_fixed_reason(setup): + c, api, *_ = setup + c.compile_files = lambda *a, **k: (_ for _ in ()).throw(ContractError("private profile value")) + status = c.reconcile_once() + serialized = canonical(api.objects[c.scope.status_name]) + assert status["reason"] == "ProfileRejected" + assert "private profile value" not in serialized + + +def test_foreign_status_is_not_adopted_and_blocks_core(setup): + c, api, _, _, calls, _ = setup + api.configmap(c.scope.status_name, {"owned": "someone else"}) + result = c.reconcile_once() + assert result["reason"] == "StatusOwnershipMismatch" and result["statusPublished"] is False + assert not calls and not api.writes + + +def test_status_conflicts_are_bounded_and_never_write_desired_maps(setup): + c, api, *_ = setup + original = copy.deepcopy({n: api.objects[n] for n in [c.scope.profile_name, c.scope.bindings_name]}) + api.conflicts = 4 + assert c.reconcile_once()["statusPublished"] is False + assert len(api.writes) == 3 + assert all(api.objects[n] == obj for n, obj in original.items()) + c.reconcile_once() + assert c.scope.status_name in api.objects + + +def test_status_update_uses_latest_cas_and_preserves_other_metadata(setup): + c, api, *_ = setup + c.reconcile_once() + obj = api.objects[c.scope.status_name] + obj["metadata"]["annotations"] = {"example.test/keep": "value"} + update_intent(setup, lambda i: i.update(generation=2, desiredState="Stopped")) + api.conflicts = 1 + c.reconcile_once() + assert api.writes[-1][1] is not None + assert api.objects[c.scope.status_name]["metadata"]["annotations"] == {"example.test/keep": "value"} + + +def test_process_lifetime_lock_blocks_second_writer_during_watch(setup): + c, api, *_ = setup + other = Controller(api, c.scope, c.store.directory.parent, c.compile_files, c.core_pass, c.verify_boundary) + + def watch(): + with pytest.raises(Busy): + other.reconcile_once() + return iter(()) + + api.on_watch = watch + c.run(max_cycles=2, sleep=lambda _: None, monotonic=lambda: 0) + assert len(api.watches) == 1 + + +def test_watch_410_relist_and_periodic_resync_are_bounded(setup): + c, api, _, _, calls, _ = setup + sleeps = [] + def expired(): + api.objects[c.scope.bindings_name]["metadata"]["resourceVersion"] = "fresh-list-version" + raise ApiError(410) + api.on_watch = expired + c.run(max_cycles=3, sleep=sleeps.append, monotonic=lambda: 0) + assert len(calls) == 3 + assert [w[1] for w in api.watches] == ["12", "fresh-list-version"] + assert sleeps == [30, 30] + + +def test_watch_event_requeues_changed_admin_intent_without_sleep(setup): + c, api, _, _, calls, _ = setup + def update(): + update_intent(setup, lambda i: i.update(generation=2, desiredState="Stopped")) + return iter([{"type": "MODIFIED", "object": api.objects[c.scope.bindings_name]}]) + api.on_watch = update + c.run(max_cycles=2, sleep=lambda _: pytest.fail("event should requeue"), monotonic=lambda: 0) + assert calls[-1][1] == {"generation": 2, "desired_state": "Stopped"} + + +def test_api_outage_has_bounded_backoff_and_no_core_mutations(setup): + c, api, _, _, calls, _ = setup + api.get = lambda _: (_ for _ in ()).throw(ApiError(503)) + assert [c.reconcile_once()["retryAfterSeconds"] for _ in range(7)] == [2, 4, 8, 16, 30, 30, 30] + assert not calls + + +@pytest.mark.parametrize("code,phase", [("UNAVAILABLE", "Retry"), ("PERMISSION_DENIED", "Blocked")]) +def test_core_rpc_failure_classification(setup, code, phase): + c, *_ = setup + c.core_pass = lambda *a, **k: (_ for _ in ()).throw(RpcError(code)) + assert c.reconcile_once()["phase"] == phase + + +@pytest.fixture +def https_client(setup, tmp_path, monkeypatch): + c = setup[0] + token = tmp_path / "token" + token.write_text("token-one") + requests = [] + monkeypatch.setattr("saw_kubernetes.controller.ssl.create_default_context", lambda **_: "verified-context") + def opener(request, **kwargs): + requests.append((request, kwargs)) + return io.BytesIO(b'{"kind":"ConfigMap"}') + client = ConfigMapClient("https://api.example.test:6443", c.scope, "/test/ca", token, opener=opener) + return client, token, requests + + +def test_https_reads_rotated_projected_token_for_every_request(https_client): + client, token, requests = https_client + client.get(client.scope.profile_name) + token.write_text("token-two") + client.get(client.scope.bindings_name) + assert [r.headers["Authorization"] for r, _ in requests] == ["Bearer token-one", "Bearer token-two"] + assert all(kwargs["context"] == "verified-context" and kwargs["timeout"] <= 45 for _, kwargs in requests) + assert _NoRedirect().redirect_request(None, None, 302, None, None, "https://elsewhere") is None + + +def test_https_client_names_and_writes_are_allowlisted(https_client): + client, _, requests = https_client + with pytest.raises(ContractError, match="allowlist"): + client.get("other-configmap") + with pytest.raises(ContractError, match="status ConfigMap"): + client.write_status({"metadata": {"namespace": client.scope.namespace, "name": client.scope.bindings_name}}) + assert not requests + + +def test_https_errors_and_oversized_responses_do_not_expose_content(https_client): + client, _, _ = https_client + client.opener = lambda *a, **kw: (_ for _ in ()).throw( + HTTPError("https://secret-url", 403, "private-details", {}, io.BytesIO(b"credentials"))) + with pytest.raises(ApiError) as error: + client.get(client.scope.profile_name) + assert str(error.value) == "Kubernetes API failure (403)" + client.opener = lambda *a, **kw: io.BytesIO(b"x" * (MAX_BODY + 1)) + with pytest.raises(ApiError): + client.get(client.scope.profile_name) + + +def test_https_watch_uses_named_field_selector_and_reports_expired_version(https_client): + client, _, requests = https_client + def expired(request, **kwargs): + requests.append((request, kwargs)) + return io.BytesIO(b'{"type":"ERROR","object":{"code":410,"message":"private"}}\n') + client.opener = expired + with pytest.raises(ApiError) as error: + list(client.watch(client.scope.bindings_name, "old", timeout_seconds=5)) + assert error.value.code == 410 and "private" not in str(error.value) + assert "fieldSelector=metadata.name%3D" + client.scope.bindings_name in requests[0][0].full_url + assert requests[0][1]["timeout"] == 10 + with pytest.raises(ContractError): + list(client.watch(client.scope.profile_name, "old")) + + +@pytest.mark.parametrize("error_object", [None, [], "private-error-details"]) +def test_malformed_watch_error_object_is_a_sanitized_api_failure(https_client, error_object): + client, _, _ = https_client + body = json.dumps({"type": "ERROR", "object": error_object}).encode() + b"\n" + client.opener = lambda *a, **kw: io.BytesIO(body) + with pytest.raises(ApiError) as error: + list(client.watch(client.scope.bindings_name, "old", timeout_seconds=5)) + assert error.value.code == 0 + assert str(error.value) == "Kubernetes API failure (0)" + + +@pytest.mark.parametrize("change", [ + lambda i: i.update(generation=True), + lambda i: i.update(schemaVersion=True), + lambda i: i["bindings"].update(namespace="foreign"), + lambda i: i["profileRef"].update(name="other"), + lambda i: i.update(unreviewed="value"), +]) +def test_invalid_admin_input_never_reaches_compiler_or_core(setup, change): + c, _, _, compiled, calls, _ = setup + update_intent(setup, change) + assert c.reconcile_once()["reason"] == "InvalidInputs" + assert not compiled and not calls + + +def test_plan_pin_fences_old_compiler_before_mutation(setup): + c, _, _, _, calls, verified = setup + update_intent(setup, lambda i: i["profileRef"].update(planSHA256="a"*64)) + c.compile_files = lambda files, bindings: {"revision": "b"*64} + assert c.reconcile_once()["reason"] == "CompiledPlanMismatch" + assert not calls and not verified + c.compile_files = lambda files, bindings: {"revision": "a"*64} + assert c.reconcile_once()["phase"] == "GenericReady" + + +def test_model_update_waits_for_confirmed_stop_and_recovers_after_restart(setup): + controller, api, _, _, _, _ = setup + calls = [] + phases = iter(["GenericReady", "Pending", "Stopped", "GenericReady", "GenericReady"]) + + def core(plan, *, generation, desired_state): + calls.append((generation, desired_state)) + return {"phase": next(phases)} + + controller.core_pass = core + update_intent(setup, lambda i: i.update(restartGeneration=0)) + assert controller.reconcile_once()["phase"] == "GenericReady" + update_intent(setup, lambda i: i.update(generation=2, restartGeneration=2)) + assert controller.reconcile_once()["phase"] == "Pending" + assert not controller.store.load().get("restartGeneration") + assert controller.reconcile_once()["phase"] == "Pending" + resumed = Controller(api, controller.scope, controller.store.directory.parent, + controller.compile_files, core, controller.verify_boundary) + assert resumed.reconcile_once()["phase"] == "GenericReady" + update_intent(setup, lambda i: i.update(generation=3)) + assert resumed.reconcile_once()["phase"] == "GenericReady" + assert calls == [(2, "Running"), (3, "Stopped"), (3, "Stopped"), (4, "Running"), (6, "Running")] diff --git a/tests/kubernetes_runtime/test_custom_inference.py b/tests/kubernetes_runtime/test_custom_inference.py new file mode 100644 index 000000000..5236a670a --- /dev/null +++ b/tests/kubernetes_runtime/test_custom_inference.py @@ -0,0 +1,72 @@ +import pytest +import yaml + +from .profile_fixtures import fixture_inputs +from saw_kubernetes import compile_profile_files, ContractError +from saw_kubernetes import provider + + +def test_custom_bom_uses_bound_model_endpoint_timeout_and_marker(): + files,cfg=fixture_inputs() + plan=compile_profile_files(files,cfg) + w,=plan["workspaces"] + assert w["providers"][0]["credentials"]=={"CUSTOM_INFERENCE_API_KEY":"saw-cred.inference.api_key"} + spec=w["sandboxes"][0]["spec"] + assert spec["command"][-1]=="fixture/model" + assert plan["customInferenceURL"]=='https://models.fixture.test/v1' + assert plan["customInferenceURL"] in spec["command"][2] and "timeoutSeconds: 300" in spec["command"][2] + + +@pytest.mark.parametrize(("binding", "error"), [ + ("browser", "matching approved OpenClaw browser binding"), + ("toolGate", "read-only tool gate"), +]) +def test_custom_browser_requires_approved_bindings(binding, error): + files, cfg = fixture_inputs() + del cfg["bindings"]["default/notebook"][binding] + with pytest.raises(ContractError, match=error): + compile_profile_files(files, cfg) + + +@pytest.mark.parametrize("bad",["https://other.example/v1","http://models.fixture.test/v1", + "https://models.fixture.test:8443/v1","https://models.fixture.test/v1?token=private", + "https://someone@models.fixture.test/v1"]) +def test_bom_cannot_override_site_approved_destination(bad): + files,cfg=fixture_inputs() + key=next(k for k in files if k.endswith("sandbox.yaml")) + doc=yaml.safe_load(files[key]);doc["spec"]["sandboxes"][0]["inference"]["baseUrl"]=bad + files[key]=yaml.safe_dump(doc) + with pytest.raises((ContractError, ValueError)): compile_profile_files(files,cfg) + + +def test_missing_approval_and_widened_proxy_policy_are_rejected(): + files,cfg=fixture_inputs() + cfg.pop("customInferenceURL") + with pytest.raises(ContractError): compile_profile_files(files,cfg) + url="https://models.fixture.test/v1" + creds={"CUSTOM_INFERENCE_API_KEY":"saw-cred.inference.api_key"} + policy={"allowHosts":["models.fixture.test"],"profiles":{},"markerPrefix":"saw-cred", + "bindings":[{"providers":["custom-inference"],"key":"api_key","secret":"inference"}]} + assert provider.check_policy("custom-inference",creds,policy,url) + for patch in ({"profiles":{"custom-inference":["elsewhere.example"]}}, + {"allowHosts":["models.fixture.test","elsewhere.example"]},{"bindings":[]}): + assert not provider.check_policy("custom-inference",creds,{**policy,**patch},url) + + +def test_gateway_profile_must_match_the_pinned_destination(tmp_path): + from saw_kubernetes import Reconciler + from .test_runtime import FakeGateway + files,cfg=fixture_inputs() + plan=compile_profile_files(files,cfg) + gateway=FakeGateway() + old=gateway.call + def call(method, request): + if method=="GetProviderProfile": + return {"profile":{"id":"custom-inference","source":"interceptor/governance", + "credentials":[{"env_vars":["CUSTOM_INFERENCE_API_KEY"],"name":"api_key"}], + "endpoints":[{"host":"foreign.example","port":443,"enforcement":"NETWORK_ENFORCEMENT_MODE_ENFORCE"}]}} + return old(method,request) + gateway.call=call + with pytest.raises(ContractError,match="approved inference boundary"): + Reconciler(gateway,tmp_path,lambda _:None).reconcile(plan,generation=1) + assert not gateway.sandboxes and not gateway.providers diff --git a/tests/kubernetes_runtime/test_gateway_ca.py b/tests/kubernetes_runtime/test_gateway_ca.py new file mode 100644 index 000000000..f236f78ec --- /dev/null +++ b/tests/kubernetes_runtime/test_gateway_ca.py @@ -0,0 +1,73 @@ +"""The controller and enrollment read one owner-bound public CA ConfigMap.""" +from datetime import datetime, timedelta, timezone + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID +import pytest + +from saw_kubernetes.gateway_ca import OWNER, configmap_path, read_public_ca + + +NAMESPACE = "saw-ctl-0123456789abcdef" +GATEWAY = "alice" +OWNER_ID = "802fe583-9329-54b6-b5b4-7be58ec171f3" + + +@pytest.fixture(scope="module") +def certificate(): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Public test CA")]) + now = datetime.now(timezone.utc).replace(tzinfo=None) + value = (x509.CertificateBuilder().subject_name(subject).issuer_name(subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256())) + return value.public_bytes(serialization.Encoding.PEM).decode("ascii") + + +def configmap(certificate): + return { + "apiVersion": "v1", + "kind": "ConfigMap", + "immutable": True, + "metadata": {"name": GATEWAY + "-gateway-ca", "namespace": NAMESPACE, + "labels": {OWNER: OWNER_ID}}, + "data": {"ca.crt": certificate}, + } + + +def test_reads_one_valid_owner_bound_public_ca(certificate): + reads = [] + result = read_public_ca(NAMESPACE, GATEWAY, OWNER_ID, + lambda path: reads.append(path) or configmap(certificate)) + assert result == certificate + assert reads == [configmap_path(NAMESPACE, GATEWAY)] + + +@pytest.mark.parametrize("change", [ + lambda obj: obj["metadata"].update(name="other-gateway-ca"), + lambda obj: obj["metadata"].update(namespace="saw-other"), + lambda obj: obj["metadata"]["labels"].update({OWNER: "foreign"}), + lambda obj: obj.update(immutable=False), + lambda obj: obj["metadata"].update(deletionTimestamp="2026-01-01T00:00:00Z"), + lambda obj: obj["metadata"].update(ownerReferences=[{"name": "foreign"}]), + lambda obj: obj["data"].update(extra="not public CA material"), + lambda obj: obj["data"].update({"ca.crt": "not a certificate"}), +]) +def test_rejects_changed_or_foreign_public_ca(certificate, change): + obj = configmap(certificate) + change(obj) + with pytest.raises(ValueError, match="public gateway CA is missing, foreign or malformed"): + read_public_ca(NAMESPACE, GATEWAY, OWNER_ID, lambda _: obj) + + +@pytest.mark.parametrize("namespace,gateway", [("../other", GATEWAY), (NAMESPACE, "../other")]) +def test_rejects_path_components_before_api_read(namespace, gateway): + reads = [] + with pytest.raises(ValueError, match="public gateway CA"): + read_public_ca(namespace, gateway, OWNER_ID, lambda path: reads.append(path)) + assert reads == [] diff --git a/tests/kubernetes_runtime/test_governance_contract.py b/tests/kubernetes_runtime/test_governance_contract.py new file mode 100644 index 000000000..41bc45193 --- /dev/null +++ b/tests/kubernetes_runtime/test_governance_contract.py @@ -0,0 +1,75 @@ +"""Execute the real Rust admission guard over the real Helm/compiler output.""" +import copy +import json +import os +import shutil +import subprocess + +import pytest + +from saw_kubernetes import compile_profile_files +from .profile_fixtures import ROOT, briefing_inputs + + +def test_compiler_harness_output_and_tampering_against_rust_guard(tmp_path): + if not shutil.which("cargo"): + if os.environ.get("SAW_REQUIRE_RUST_TESTS") == "1": + pytest.fail("cargo is required by this CI job") + pytest.skip("cargo is required for the compiler/governance contract test") + files, config = briefing_inputs() + valid = [] + for additional in (0, 16): + slot = config["bindings"]["personal-assistant/assistant"] + slot["mounts"] = [{"claimName":f"data-{i}", "uid":f"uid-{i}", "readOnly":True, + "mountPath":f"/data/{i}"} for i in range(additional)] + driver = compile_profile_files(files, config)["workspaces"][0]["sandboxes"][0]["spec"]["template"]["driver_config"] + assert len(driver["kubernetes"]["volumes"]) == 3 + additional + assert len(driver["kubernetes"]["containers"]["agent"]["volume_mounts"]) == 5 + additional + valid.append(driver) + invalid = [] + for index, field, value in [ + (3,"name","saw-workspace"), (3,"read_only",False), (3,"mount_path","/etc"), + (3,"sub_path","bundle"), (3,"sub_path","harness-../outside"), + (3,"sub_path","harness-"+"g"*64), (3,"sub_path","harness-"+"a"*63), + (3,"extra",True), (4,"name","saw-audit"), (4,"sub_path","other"), + (4,"read_only",True), (4,"mount_path","/sandbox/harness"), + ]: + driver = copy.deepcopy(valid[0]) + driver["kubernetes"]["containers"]["agent"]["volume_mounts"][index][field] = value + invalid.append(driver) + for count in (1, 3): + driver = copy.deepcopy(valid[0]) + mounts = driver["kubernetes"]["containers"]["agent"]["volume_mounts"] + if count == 1: + mounts.pop() + else: + mounts.append(copy.deepcopy(mounts[-1])) + invalid.append(driver) + extension = ROOT/"image-builder-charts/helm/governance-interceptor-image/files/saw-extension/driver_config.rs" + (tmp_path/"src").mkdir() + (tmp_path/"Cargo.toml").write_text('[package]\nname="compiler-guard-contract"\nversion="0.1.0"\nedition="2021"\n[dependencies]\nserde_json="1"\n') + (tmp_path/"src/lib.rs").write_text(""" +use std::collections::HashMap; +use serde_json::Value; +struct InterceptorResult { allowed: bool } +fn allow() -> InterceptorResult { InterceptorResult { allowed: true } } +fn deny(_: &str) -> InterceptorResult { InterceptorResult { allowed: false } } +mod driver { +""" + extension.read_text() + """ +#[test] +fn compiled_harness_contract() { + let principal = HashMap::from([("kind".into(),"user".into()), ("provider".into(),"mtls".into()), + ("roles".into(),"openshell-admin".into())]); +""" + '\n'.join( + ' assert_eq!(validate_driver_config(Some(&serde_json::from_str::(r#"' + + json.dumps(driver) + '"#).unwrap()), &principal).allowed, ' + str(allowed).lower() + ');' + for allowed, drivers in [(True, valid), (False, invalid)] for driver in drivers + ) + "\n}\n}\n") + result = subprocess.run(["cargo","test","--offline","--manifest-path",str(tmp_path/"Cargo.toml")], + capture_output=True, text=True) + if result.returncode and ("no matching package named" in result.stderr or + "attempting to make an HTTP request, but --offline was specified" in result.stderr): + if os.environ.get("SAW_REQUIRE_RUST_TESTS") == "1": + pytest.fail("fetch serde_json before running the required Rust contract test") + pytest.skip("serde_json is not cached; run with the CI Rust toolchain") + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/kubernetes_runtime/test_identity_bootstrap.py b/tests/kubernetes_runtime/test_identity_bootstrap.py new file mode 100644 index 000000000..dc4d27e82 --- /dev/null +++ b/tests/kubernetes_runtime/test_identity_bootstrap.py @@ -0,0 +1,239 @@ +"""Create-once identity and crash recovery; fake API, real OpenSSL leaf issuance.""" +import copy +import importlib.util +from pathlib import Path +import shutil +import subprocess + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("identity_bootstrap", ROOT / "charts/saw-kubernetes-identity/files/bootstrap.py") +boot = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(boot) + + +@pytest.fixture +def config(): + return dict(ownerID="67a9891d-e281-442f-86c3-41771f5ce049", namespace="saw-test-trusted", + targetNamespace="saw-test", gateway="saw-test", + serverSans=["saw-test-gateway.saw-test.svc", "saw-test-gateway.saw-test.svc.cluster.local"]) + + +def material(prefix="test"): + data = {key: boot.encode((prefix + key).encode()) for key in + ("ca", "serverCert", "serverKey", "signing", "public", "kid")} + data["kek"] = boot.encode(boot.encode(b"a" * 32).encode()) + return data + + +class Api: + def __init__(self, config): + self.namespaces = {name: {"metadata": {"uid": name + "-uid", "labels": {boot.OWNER: config["ownerID"]}}} + for name in (config["namespace"], config["targetNamespace"])} + self.values, self.created = {}, [] + self.fail_after = None + self.discovery = { + "agents.x-k8s.io/v1beta1": [ + {"name": "sandboxes", "kind": "Sandbox", "namespaced": True}], + "policy.networking.k8s.io/v1alpha1": [ + {"name": "adminnetworkpolicies", "kind": "AdminNetworkPolicy", "namespaced": False}], + } + + def request(self, method, path, body=None): + if path.startswith("/apis/"): + assert method == "GET" + group = path.removeprefix("/apis/") + if group in self.discovery: + return {"groupVersion": group, "resources": self.discovery[group]} + return None + parts = path.split("/") + ns = parts[4] + assert parts[5] == "configmaps" + if method == "GET": + return self.get(ns, parts[6]) + assert method == "POST" + return self.create(ns, body) + + def namespace(self, name): + return copy.deepcopy(self.namespaces[name]) + + def get(self, ns, name): + return copy.deepcopy(self.values.get((ns, name))) + + def create(self, ns, body): + key = ns, body["metadata"]["name"] + if key not in self.values: + self.values[key] = copy.deepcopy(body) + self.created.append(key) + if self.fail_after == len(self.created): + raise boot.Failure("simulated ambiguous create") + + +def never(): + pytest.fail("existing retained bundle must not regenerate identity") + + +@pytest.mark.parametrize("group", ["agents.x-k8s.io/v1beta1", "policy.networking.k8s.io/v1alpha1"]) +@pytest.mark.parametrize("defect", ["absent", "wrong-kind", "wrong-scope", "subresource-only"]) +def test_live_api_must_match_before_any_identity_is_written(config, group, defect): + api = Api(config) + if defect == "absent": + del api.discovery[group] + else: + entry = api.discovery[group][0] + if defect == "wrong-kind": + entry["kind"] = "Other" + elif defect == "wrong-scope": + entry["namespaced"] = not entry["namespaced"] + else: + entry["name"] += "/status" + with pytest.raises(boot.Failure, match="required Kubernetes API is unavailable"): + boot.run(api, config, never) + assert not api.created + + +def test_repeat_sync_preserves_identity_without_retained_administrator(config): + api = Api(config) + boot.run(api, config, material) + before = copy.deepcopy(api.values) + boot.run(api, config, never) + assert before == api.values and len(api.created) == 6 + runtime = [v for (ns, _), v in api.values.items() if ns == config["targetNamespace"]] + assert len(runtime) == 4 + assert not any("admin" in name.lower() for v in api.values.values() for name in v["data"]) + for (_, name), value in api.values.items(): + assert value["immutable"] is (not name.endswith("-client-ca")) + assert not value["metadata"].get("ownerReferences") + public = api.get(config["namespace"], "saw-test-gateway-ca") + assert public["kind"] == "ConfigMap" and set(public["data"]) == {"ca.crt"} + assert not any("ca.key" in v["data"] for v in api.values.values()) + + +@pytest.mark.parametrize("point", range(1, 7)) +def test_resume_after_ambiguous_create(config, point): + api = Api(config) + api.fail_after = point + with pytest.raises(boot.Failure): + boot.run(api, config, material) + retained = api.get(config["namespace"], "saw-test-identity") + api.fail_after = None + boot.run(api, config, never) + assert api.get(config["namespace"], "saw-test-identity") == retained + assert len(api.created) == 6 + + +def test_racing_creator_uses_atomic_winner(config): + api = Api(config) + create = api.create + def winner(ns, body): + if body["metadata"]["name"].endswith("-identity"): + body = copy.deepcopy(body) + body["data"] = material("winner") + create(ns, body) + api.create = winner + boot.run(api, config, material) + assert api.get(config["targetNamespace"], "saw-test-server")["data"]["tls.key"] == material("winner")["serverKey"] + + +def test_orphaned_projection_never_regenerates(config): + api = Api(config) + api.values[(config["targetNamespace"], "saw-test-server")] = {"data": {}} + with pytest.raises(boot.Failure, match="orphaned"): + boot.run(api, config, never) + assert not api.created + + +@pytest.mark.parametrize("mutation", ["uid", "owner", "san", "deleting", "data", "gc", "mutable"]) +def test_foreign_or_changed_identity_fails_without_mutation(config, mutation): + api = Api(config) + boot.run(api, config, material) + retained = api.values[(config["namespace"], "saw-test-identity")] + if mutation == "uid": + api.namespaces[config["targetNamespace"]]["metadata"]["uid"] = "replacement" + elif mutation == "owner": + retained["metadata"]["labels"][boot.OWNER] = "other" + elif mutation == "san": + config["serverSans"].append("new.example.test") + elif mutation == "deleting": + retained["metadata"]["deletionTimestamp"] = "now" + elif mutation == "data": + del retained["data"]["serverKey"] + elif mutation == "gc": + retained["metadata"]["ownerReferences"] = [{"kind": "Job"}] + elif mutation == "mutable": + retained["immutable"] = False + with pytest.raises(boot.Failure): + boot.run(api, config, never) + assert len(api.created) == 6 + + +def test_conflicting_projection_is_not_overwritten(config): + api = Api(config) + boot.run(api, config, material) + api.values[(config["targetNamespace"], "saw-test-kek")]["data"]["key-encryption-key"] = "foreign" + with pytest.raises(boot.Failure, match="foreign"): + boot.run(api, config, never) + assert len(api.created) == 6 + + +@pytest.mark.parametrize("field,value", [("targetNamespace", "saw-test-trusted"), ("ownerID", "not-uuid"), + ("serverSans", ["https://bad"]), ("serverSans", []), ("adminDays", True), ("adminDays", 91), + ("gateway", "../../bad"), ("namespace", "bad/name")]) +def test_invalid_inputs_fail_before_api(config, field, value): + config[field] = value + with pytest.raises(boot.Failure): + boot.run(None, config, never) + + +@pytest.mark.skipif(not shutil.which("openssl"), reason="OpenSSL required") +def test_real_stock_server_identity_has_no_administrator_key(config, tmp_path, capsys): + def command(*args): + subprocess.run(["openssl", *map(str, args)], capture_output=True, check=True) + (tmp_path / "server").mkdir() + (tmp_path / "jwt").mkdir() + command("req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", + "-keyout", tmp_path / "ca.key", "-out", tmp_path / "ca.crt", "-days", "2", "-subj", "/CN=test-ca") + command("req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", + "-keyout", tmp_path / "server/tls.key", "-out", tmp_path / "server/request", "-subj", "/CN=gateway") + (tmp_path / "server/ext").write_text("subjectAltName=" + ",".join("DNS:" + n for n in config["serverSans"]) + "\n") + command("x509", "-req", "-in", tmp_path / "server/request", "-CA", tmp_path / "ca.crt", + "-CAkey", tmp_path / "ca.key", "-set_serial", "2", "-days", "2", "-extfile", tmp_path / "server/ext", + "-out", tmp_path / "server/tls.crt") + for name in ("signing.pem", "public.pem", "kid"): + (tmp_path / "jwt" / name).write_text("synthetic-jwt-fixture") + data = boot.generate(config, tmp_path) + assert not (tmp_path / "ca.key").exists() + assert not (tmp_path / "admin").exists() + assert not {"adminKey", "adminCert"}.intersection(data) + assert len(boot.decode(boot.decode(data["kek"]))) == 32 + assert not capsys.readouterr().out + + +def test_orphaned_public_ca_never_regenerates(config): + api = Api(config) + api.values[(config["namespace"], "saw-test-gateway-ca")] = {"data": {}} + with pytest.raises(boot.Failure, match="orphaned"): + boot.run(api, config, never) + assert not api.created + + +def test_existing_admin_bundle_is_rejected_before_new_projections(config): + api = Api(config) + boot.run(api, config, material) + api.values[(config["namespace"], "saw-test-identity")]["data"]["adminKey"] = boot.encode(b"obsolete") + with pytest.raises(boot.Failure, match="incomplete"): + boot.run(api, config, never) + assert len(api.created) == 6 + + +def test_repeat_bootstrap_does_not_remove_active_public_session(config): + api = Api(config) + boot.run(api, config, material) + trust = api.values[(config["targetNamespace"], "saw-test-client-ca")] + trust["data"]["session-67a9891d-e281-442f-86c3-41771f5ce049.crt"] = boot.encode(b"public-certificate") + trust["data"]["ca.crt"] = boot.encode(b"active-public-bundle") + trust["metadata"]["annotations"]["saw.redhat.com/admin-sessions"] = "fixture" + before = copy.deepcopy(api.values) + boot.run(api, config, never) + assert api.values == before diff --git a/tests/kubernetes_runtime/test_inference_providers.py b/tests/kubernetes_runtime/test_inference_providers.py new file mode 100644 index 000000000..f082cf30d --- /dev/null +++ b/tests/kubernetes_runtime/test_inference_providers.py @@ -0,0 +1,83 @@ +"""All inference providers share BOM resolution and the VM's proxy profiles.""" +import json +import shutil +import subprocess + +import pytest + +from saw_kubernetes import ContractError, compile_profile +from saw_kubernetes import provider as governed +from .test_openclaw import agent +from .test_runtime import source, sandbox + + +CASES = [ + ("openai", "gpt-4.1-mini", "https://api.openai.com/v1", "openai-completions", "OPENAI_API_KEY"), + ("anthropic", "claude-sonnet-4-6", "https://api.anthropic.com", "anthropic-messages", "ANTHROPIC_API_KEY"), + ("nvidia", "nvidia/nemotron-3.5-lightning-30b-a3b", "https://integrate.api.nvidia.com/v1", "openai-completions", "NVIDIA_API_KEY"), +] + + +@pytest.mark.parametrize("kind,model,url,api,env", CASES) +@pytest.mark.parametrize("native", [True, False]) +def test_provider_compiles_native_protocol_and_placeholder_config(source, kind, model, url, api, env, native): + agent(source, kind, model) + if native: + sandbox(source, lambda s: s["inference"].pop("baseUrl")) + workspace = compile_profile(*source)["workspaces"][0] + assert workspace["providers"][0]["credentials"] == {env:"saw-cred.inference.api_key"} + command = workspace["sandboxes"][0]["spec"]["command"] + assert "@SAW_" not in command[2] + if not shutil.which("node"): + pytest.skip("node is required to execute the compiled OpenClaw configuration") + javascript = command[2].split("<<'JS'\n", 1)[1].split("\nJS\n", 1)[0] + # Execute the generated startup without writing /sandbox or starting an agent. + runner = """ +const vm = require('node:vm'); +let written; +vm.runInNewContext(process.argv[1], { + require: name => name === 'node:fs' ? { + writeFileSync: (path, data) => { written = data; }, renameSync: () => {} + } : require(name), + process: {argv: ['node','-',process.argv[2]], env:{OPENCLAW_CONFIG_PATH:'/fixture/config'}} +}); +console.log(written); +""" + result = subprocess.run(["node", "-e", runner, javascript, model], capture_output=True, text=True, check=True) + batch = json.loads(result.stdout) + config = {operation["path"]: operation["value"] for operation in batch} + assert set(config) == {"gateway", "models", "agents", "tools", "plugins", "skills", "channels", "mcp"} + assert all(set(operation) == {"path", "value"} for operation in batch) + # Whole managed subtrees replace previous settings; the stock writer owns + # metadata/backup publication and unrelated user configuration remains. + assert 'openclaw config set --batch-file "$OPENCLAW_CONFIG_PATH.saw-managed"' in command[2] + actual = config["models"]["providers"]["saw-" + kind] + assert actual == {"baseUrl":url,"api":api,"apiKey":"${"+env+"}","models":[{"id":model,"name":model}]} + assert config["agents"]["defaults"]["model"]["primary"] == "saw-"+kind+"/"+model + assert config["tools"] == {"deny":["*"]} + assert config["plugins"] == {"allow":[], "load":{"paths":[]}, "entries":{}} + assert config["skills"] == {"load":{"extraDirs":[]}} + assert config["channels"] == {} and config["mcp"] == {"servers":{}} + + +@pytest.mark.parametrize("kind,model,url,api,env", CASES) +def test_provider_uses_shipped_host_and_credential_policy(source, kind, model, url, api, env): + agent(source, kind, model) + credentials = compile_profile(*source)["workspaces"][0]["providers"][0]["credentials"] + slot, hosts = governed.requirements(kind) + assert slot["env_vars"] == [env] + policy = {"markerPrefix":"saw-cred", "profiles":{kind:hosts}, "allowHosts":hosts, + "bindings":[{"providers":[kind],"secret":"inference","key":"api_key"}]} + assert governed.check_policy(kind, credentials, policy) + assert not governed.check_policy(kind, credentials, {**policy,"allowHosts":[]}) + assert not governed.check_policy(kind, {env:"saw-cred.foreign.api_key"}, policy) + sandbox(source, lambda s: s["inference"].update(baseUrl="https://foreign.example/v1")) + with pytest.raises(ContractError): + compile_profile(*source) + + +def test_anthropic_cannot_silently_use_openai_protocol(source): + agent(source, "anthropic", "claude-sonnet-4-6") + sandbox(source, lambda s: s["inference"].update(protocol="openai")) + with pytest.raises(ContractError): + compile_profile(*source) diff --git a/tests/kubernetes_runtime/test_intent.py b/tests/kubernetes_runtime/test_intent.py new file mode 100644 index 000000000..fd008580f --- /dev/null +++ b/tests/kubernetes_runtime/test_intent.py @@ -0,0 +1,247 @@ +"""Pure local intent generation; no network, Kubernetes API or credential access.""" + +import copy +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import yaml + +from saw_kubernetes.compiler import OWNER, canonical, digest +from saw_kubernetes.intent import IntentError, build_intent +from .test_runtime import flat_profile, source + + +@pytest.fixture +def inputs(source): + profile = {"apiVersion": "v1", "kind": "ConfigMap", "metadata": { + "name": "saw-alice-bom", "namespace": "saw-alice", + "uid": "c8c8a0c0-063b-4412-b196-2788bf732cac", "resourceVersion": "opaque-rv"}, + "data": flat_profile(source)} + return profile, source[1] + + +def build(inputs, **kwargs): + return build_intent(*inputs, name="saw-runtime-intent", generation=kwargs.pop("generation", 1), **kwargs) + + +def test_builder_outputs_exact_controller_envelope_without_mutating_inputs(inputs): + original = copy.deepcopy(inputs) + result = build(inputs) + assert result["metadata"] == {"name": "saw-runtime-intent", "namespace": "saw-alice", + "labels": {OWNER: inputs[1]["ownerID"]}} + assert set(result["data"]) == {"intent.json"} + intent = json.loads(result["data"]["intent.json"]) + assert intent == {"schemaVersion": 1, "generation": 1, "desiredState": "Running", + "profileRef": {"name": "saw-alice-bom", "uid": inputs[0]["metadata"]["uid"], + "sha256": digest(inputs[0]["data"])}, "bindings": inputs[1]} + assert result["data"]["intent.json"] == canonical(intent) + assert inputs == original + + +def test_hash_covers_exact_data_strings_not_metadata_or_yaml_reserialization(inputs): + first = json.loads(build(inputs)["data"]["intent.json"])["profileRef"]["sha256"] + inputs[0]["metadata"].update(resourceVersion="new", annotations={"arbitrary": "not included"}) + assert json.loads(build(inputs)["data"]["intent.json"])["profileRef"]["sha256"] == first + key = next(iter(inputs[0]["data"])) + inputs[0]["data"][key] += "\n" + assert json.loads(build(inputs)["data"]["intent.json"])["profileRef"]["sha256"] != first + + +@pytest.mark.parametrize("change", [ + lambda p, b: p["metadata"].pop("uid"), + lambda p, b: p["metadata"].update(uid=""), + lambda p, b: p["metadata"].pop("resourceVersion"), + lambda p, b: p["metadata"].update(deletionTimestamp="now"), + lambda p, b: p["metadata"].update(namespace="other"), + lambda p, b: p.update(kind="Secret"), + lambda p, b: p.update(binaryData={"payload": "encoded"}), + lambda p, b: p["data"].update({"../file": "content"}), + lambda p, b: b.update(ownerID=""), +]) +def test_missing_observation_or_invalid_scope_rejected(inputs, change): + change(*inputs) + with pytest.raises(IntentError): + build(inputs) + + +@pytest.mark.parametrize("generation", [0, -1, True, "1"]) +def test_generation_must_be_a_positive_integer(inputs, generation): + with pytest.raises(IntentError): + build(inputs, generation=generation) + + +@pytest.mark.parametrize("bad_name", ["a.-b", "a-.b", "a..b", "a.-.b"]) +def test_builder_never_emits_an_invalid_dns_subdomain_name(inputs, bad_name): + with pytest.raises(IntentError, match="name must be valid"): + build_intent(*inputs, name=bad_name, generation=1) + + +def test_builder_accepts_valid_dotted_intent_name(inputs): + result = build_intent(*inputs, name="runtime-intent.prod-1", generation=1) + assert result["metadata"]["name"] == "runtime-intent.prod-1" + + +def test_complete_profile_is_compiled_before_output(inputs): + key = next(k for k in inputs[0]["data"] if k.endswith("sandbox.yaml")) + doc = yaml.safe_load(inputs[0]["data"][key]) + doc["spec"]["sandboxes"][0]["type"] = "openclaw" + inputs[0]["data"][key] = yaml.safe_dump(doc) + with pytest.raises(IntentError, match="unsupported or invalid"): + build(inputs) + + +def test_previous_intent_allows_exact_retry_but_requires_new_generation_for_change(inputs): + previous = build(inputs) + assert build(inputs, previous=previous) == previous + with pytest.raises(IntentError, match="reused"): + build(inputs, desired_state="Stopped", previous=previous) + stopped = build(inputs, generation=2, desired_state="Stopped", previous=previous) + assert json.loads(stopped["data"]["intent.json"])["desiredState"] == "Stopped" + with pytest.raises(IntentError, match="rolled back"): + build(inputs, previous=stopped) + + +def test_previous_intent_cannot_hide_same_name_profile_uid_replacement(inputs): + previous = build(inputs) + inputs[0]["metadata"]["uid"] = "8ef68f29-ac12-4836-a8d7-e0d969672c53" + with pytest.raises(IntentError, match="explicit controller recovery"): + build(inputs, generation=2, previous=previous) + + +def test_previous_intent_from_another_owner_is_not_accepted(inputs): + previous = build(inputs) + previous["metadata"]["labels"][OWNER] = "someone-else" + with pytest.raises(IntentError, match="another controller scope"): + build(inputs, previous=previous) + + +@pytest.fixture +def cli(inputs, tmp_path): + profile, bindings = tmp_path / "observed-profile.json", tmp_path / "admin-bindings.yaml" + profile.write_text(json.dumps(inputs[0])) + bindings.write_text(yaml.safe_dump(inputs[1])) + root = Path(__file__).resolve().parents[2] + env = dict(os.environ, PYTHONPATH=str(root / "kubernetes-runtime"), PYTHONDONTWRITEBYTECODE="1") + command = [sys.executable, "-m", "saw_kubernetes.intent", "--profile-configmap", str(profile), + "--bindings", str(bindings), "--name", "saw-runtime-intent", "--generation", "1", + "--desired-state", "Running"] + return command, env, profile, bindings + + +def test_cli_writes_only_validated_json_to_stdout(inputs, cli): + command, env, profile, bindings = cli + before = {p: p.read_bytes() for p in [profile, bindings]} + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 0 and result.stderr == "" + assert json.loads(result.stdout) == build(inputs) + assert all(path.read_bytes() == raw for path, raw in before.items()) + + +def test_cli_errors_are_sanitized_with_empty_stdout(cli): + command, env, profile, _ = cli + profile.write_text('{"private-secret-token": invalid-json}') + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 1 and result.stdout == "" + assert "private-secret-token" not in result.stderr and "Traceback" not in result.stderr + + +def test_cli_rejects_duplicate_json_keys(cli): + command, env, profile, _ = cli + raw = profile.read_text() + profile.write_text('{"kind":"ConfigMap",' + raw[1:]) + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 1 and result.stdout == "" + + +def test_cli_rejects_duplicate_yaml_binding_keys(cli): + command, env, _, bindings = cli + bindings.write_text(bindings.read_text() + "\nnamespace: saw-alice\n") + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 1 and result.stdout == "" + + +def test_cli_argument_errors_do_not_echo_supplied_values(cli): + command, env, _, _ = cli + command[command.index("--generation") + 1] = "private-misplaced-value" + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 1 and result.stdout == "" + assert result.stderr == "invalid command-line arguments\n" + + +def test_cli_previous_intent_update_validation(inputs, cli, tmp_path): + command, env, _, _ = cli + previous = tmp_path / "previous-intent.json" + previous.write_text(json.dumps(build(inputs, generation=2))) + result = subprocess.run(command + ["--previous-intent", str(previous)], env=env, capture_output=True, text=True) + assert result.returncode == 1 and result.stdout == "" + assert "generation" in result.stderr + + +def test_restart_generation_round_trip_and_rollback(inputs): + first = build(inputs, generation=2, restart_generation=2) + retry = build(inputs, generation=2, previous=first) + assert retry == first + update = build(inputs, generation=3, previous=first, desired_state="Stopped") + assert json.loads(update["data"]["intent.json"])["restartGeneration"] == 2 + for value in [True, -1, 1, 4]: + with pytest.raises(IntentError, match="restart generation"): + build(inputs, generation=3, restart_generation=value, previous=first) + legacy = build(inputs) + assert "restartGeneration" not in json.loads(build(inputs, generation=2, previous=legacy)["data"]["intent.json"]) + + +def test_helm_lifecycle_intent_round_trips_through_cli_and_controller(source, inputs, tmp_path, capsys): + import base64 + from types import SimpleNamespace + from saw_kubernetes.compiler import compile_profile_files + from saw_kubernetes.controller import Controller, Scope + from saw_kubernetes.intent import main + from .test_openclaw import agent + + root = Path(__file__).resolve().parents[2] + profile, bindings = inputs + agent(source) + profile["metadata"]["name"] = "saw-bom-profiles" + profile["data"] = flat_profile(source) + site = yaml.safe_load((root / "tests/fixtures/container-site.yaml").read_text())["portal"]["containerSites"]["alice"] + bindings["ownerSubject"] = site["subject"] + slot = bindings["bindings"]["default/toolbox"] + slot["workspace"]["claimName"] = "workspace-alice" + slot["toolGate"] = { + "ownerLogin": "alice", "receiptKey": base64.b64encode(bytes.fromhex("302a300506032b6570032100") + bytes(32)).decode(), + "trusted": {"claimName": "saw-gate-alice", "uid": "gate-uid", "readOnly": True}, + "audit": {"claimName": "saw-audit-alice", "uid": "audit-uid", "readOnly": False}} + original = build(inputs, generation=2, restart_generation=1, pin_plan=True) + reg = "a8cb6cbd0eb62f31" + auto = site["automaticEnrollment"] + site.update(lifecycleVersion=1, restartGeneration=1, + targets={"default/toolbox": dict(workspace="default", sandbox="toolbox", ui=False, base="alice", + relayID=reg, subjectMap="saw-portal-relay-subject")}, + enrollment=dict(registration=reg, revision=auto["revision"], image=auto["image"], + buildNamespace=auto["buildNamespace"], approved={}, + intent=json.loads(original["data"]["intent.json"]))) + values = {"site": site, "owner": "alice", "ownerSubject": site["subject"], "installationID": bindings["ownerID"], + "registration": reg, "intentGeneration": 2, "vaultPrefix": "secret/data/hub/saw-alice/"+reg, + "global": {"repoURL":"https://example.test/repo", "targetRevision":"test", + "clusterDomain":"cluster.example.test", "vpArgoNamespace":"vp-gitops"}} + rendered = subprocess.run(["helm","template","alice",str(root/"charts/saw-portal-container"), + "--namespace","saw-alice","-f","-"], input=json.dumps(values), + capture_output=True, text=True, check=True) + previous, = [obj for obj in yaml.safe_load_all(rendered.stdout) if obj and obj["kind"] == "ConfigMap" + and obj["metadata"]["name"] == "saw-runtime-intent"] + for name, data in (("profile",profile), ("bindings",bindings), ("previous",previous)): + (tmp_path/(name+".json")).write_text(json.dumps(data)) + assert main(["--profile-configmap",str(tmp_path/"profile.json"),"--bindings",str(tmp_path/"bindings.json"), + "--previous-intent",str(tmp_path/"previous.json"),"--name","saw-runtime-intent", + "--generation","2","--pin-plan"]) == 0 + replay = json.loads(capsys.readouterr().out) + assert json.loads(replay["data"]["intent.json"]) == json.loads(previous["data"]["intent.json"]) + replay["metadata"].update(uid="intent-uid", resourceVersion="2") + api = SimpleNamespace(get={"saw-bom-profiles":profile,"saw-runtime-intent":replay}.get) + scope = Scope("saw-alice","saw-bom-profiles","saw-runtime-intent","saw-runtime-status", bindings["ownerID"]) + ctl = Controller(api, scope, tmp_path, compile_profile_files, lambda *args: None, lambda plan: None) + assert ctl._snapshot()[0]["restartGeneration"] == 1 diff --git a/tests/kubernetes_runtime/test_model_selection.py b/tests/kubernetes_runtime/test_model_selection.py new file mode 100644 index 000000000..a64ec2bea --- /dev/null +++ b/tests/kubernetes_runtime/test_model_selection.py @@ -0,0 +1,21 @@ +import json + +import pytest + +from saw_kubernetes.prepare_gate import publish_model, verify + + +def test_publication_fences_stale_and_conflicting_jobs(tmp_path): + bundle = tmp_path / "bundle" + bundle.mkdir() + for generation, model in [(1, "first"), (2, "second"), (1, "first"), (2, "second")]: + publish_model(tmp_path, bundle, model, generation) + assert json.loads((bundle / "model.json").read_text()) == {"generation": 2, "model": "second"} + verify(bundle, {}, model_selector=True) + with pytest.raises(ValueError, match="conflicting"): + publish_model(tmp_path, bundle, "different", 2) + with pytest.raises(ValueError, match="invalid"): + publish_model(tmp_path, bundle, "../outside", 3) + (bundle / "unexpected").touch() + with pytest.raises(ValueError): + verify(bundle, {}, model_selector=True) diff --git a/tests/kubernetes_runtime/test_openclaw.py b/tests/kubernetes_runtime/test_openclaw.py new file mode 100644 index 000000000..dff64aeda --- /dev/null +++ b/tests/kubernetes_runtime/test_openclaw.py @@ -0,0 +1,185 @@ +import json +import pytest + +from saw_kubernetes.compiler import ContractError, compile_profile +from saw_kubernetes.reconcile import Reconciler +from .test_runtime import FakeGateway, mutate, sandbox, source + + +def agent(source, kind="openai", model="gpt-4.1-mini-2025-04-14"): + from saw_kubernetes.core import shared + protocol = "openai" if kind == "custom-inference" else shared.PROVIDER_PROTOCOLS[kind] + endpoint = ("https://models.example.test/v1" if kind == "custom-inference" else + shared.NATIVE_BASE_URLS.get(kind) or shared.PROTOCOL_NATIVE_BASE_URLS[protocol]) + if kind == "custom-inference": + source[1]["customInferenceURL"] = endpoint + source[1]["proxyBindings"] = [{"type":kind, "secret":"inference", "key":"api_key", "refresh":False}] + mutate(source, "providers.yaml", lambda d: d["spec"]["providers"].append( + {"name": kind, "type": kind, "credentialSecret": "inference"})) + sandbox(source, lambda s: s.update(type="openclaw", providers=[kind], inference={ + "providerRef": kind, "protocol": protocol, "baseUrl": endpoint, "model": model})) + + +@pytest.mark.parametrize("kind,model", [ + ("openai", "gpt-4.1-mini"), ("anthropic", "claude-sonnet-4-6"), + ("nvidia", "nvidia/nemotron-3.5-lightning-30b-a3b"), ("custom-inference", "org/model"), +]) +def test_mutable_model_compiles_and_startup_accepts_published_selection(source, kind, model): + import base64 + import shutil + import subprocess + agent(source, kind, model) + binding = source[1]["bindings"]["default/toolbox"] + key = base64.b64encode(bytes.fromhex("302a300506032b6570032100") + bytes(range(32))).decode() + binding["toolGate"] = {"ownerLogin":"alice", "receiptKey":key, "modelSelector":True, + "trusted":{"claimName":"saw-gate-test","uid":"gate-uid","readOnly":True}, + "audit":{"claimName":"saw-audit-test","uid":"audit-uid","readOnly":False}} + command = compile_profile(*source)["workspaces"][0]["sandboxes"][0]["spec"]["command"] + assert command[-1] == "model-from-gate" + # Execute the actual startup selection, including the provider's model syntax. + selection = command[2][command[2].index("const selection"):command[2].index("const config")] + if not shutil.which("node"): + pytest.skip("node is required to execute the compiled startup validation") + for value, valid in [(model, True), ("$(id)", False), ("../escape", False)]: + js = "const fs={readFileSync:()=>" + json.dumps(json.dumps({"generation":1,"model":value})) + "};\n" + result = subprocess.run(["node", "-e", js + selection + "console.log(model);"], capture_output=True, text=True) + assert (result.returncode == 0) == valid, result.stderr + if valid: + assert result.stdout.strip() == model + + +def test_openclaw_pinned_proxy_contract(source): + agent(source) + ws = compile_profile(*source)["workspaces"][0] + assert ws["providers"][0]["credentials"] == {"OPENAI_API_KEY": "saw-cred.inference.api_key"} + command = ws["sandboxes"][0]["spec"]["command"] + assert command[-1] == "gpt-4.1-mini-2025-04-14" + assert "saw-cred" not in json.dumps(command) + assert "loopback" in command[2] + + +@pytest.mark.parametrize("change", ["absent", "alternative-url", "shell", "unknown-input"]) +def test_openclaw_rejects_unapproved_bindings(source, change): + agent(source) + if change == "absent": + source[1].pop("proxyBindings") + else: + update = {"alternative-url": {"baseUrl": "https://example.com/v1"}, + "shell": {"model": "$(id)"}, "unknown-input": {"model": {"secretKeyRef": {"name":"private","key":"key"}}}} + sandbox(source, lambda sb: sb["inference"].update(update[change])) + with pytest.raises(ContractError): + compile_profile(*source) + + +def test_redacted_provider_refuses_changed_revision(source, tmp_path): + agent(source) + plan = compile_profile(*source) + ws = plan["workspaces"][0] + provider = ws["providers"][0] + gateway = FakeGateway() + core = Reconciler(gateway, tmp_path / "ledger", lambda plan: None) + real_call = gateway.call + def call(method, body): + result = real_call(method, body) + if method == "CreateProvider": + obj = result["provider"] + obj["metadata"]["resource_version"] = "1" + obj["credentials"]["OPENAI_API_KEY"] = "REDACTED" + return result + gateway.call = call + state = {} + assert not core._apply_provider(ws, provider, plan, state, 1, []) + assert core._apply_provider(ws, provider, plan, state, 1, []) + gateway.providers[(ws["name"], "openai")]["metadata"]["resource_version"] = "2" + with pytest.raises(ContractError, match="revision"): + core._apply_provider(ws, provider, plan, state, 1, []) + + +def test_retirement_checks_recorded_id_before_mutations(source, tmp_path): + import uuid + source[1]["retiredSandboxes"] = {"agent/old": str(uuid.uuid4())} + plan = compile_profile(*source) + gateway = FakeGateway() + core = Reconciler(gateway, tmp_path / "ledger", lambda plan: None) + with pytest.raises(ContractError, match="recorded identity"): + core.reconcile(plan, generation=1) + assert not gateway.sandboxes + + +def test_browser_policy_is_bound_to_one_owner_and_exact_https_origin(source): + agent(source) + binding = next(iter(source[1]["bindings"].values())) + binding["browser"] = {"origin":"https://agent.apps.example.com","ownerLogin":"alice"} + with pytest.raises(ContractError, match="browser requires the read-only tool gate"): + compile_profile(*source) + binding["toolGate"] = {"ownerLogin": "alice", + "trusted": {"claimName": "saw-gate-test", "uid": "gate-uid", "readOnly": True}, + "audit": {"claimName": "saw-audit-test", "uid": "audit-uid", "readOnly": False}} + ws = compile_profile(*source)["workspaces"][0] + command = ws["sandboxes"][0]["spec"]["command"][2] + assert 'x-saw-relay-identity' in command and source[1]['ownerSubject'] in command + assert '/sandbox/.saw/trusted/relay-proof' in command + assert 'allowedOrigins:["https://agent.apps.example.com"]' in command + assert command.index("config.gateway.trustedProxies") < command.index("fs.writeFileSync") + binding["browser"]["origin"] = "https://*.example.com" + with pytest.raises(ContractError): + compile_profile(*source) + + +def test_tool_gate_uses_shared_identity_policy_and_exact_mounts(source): + from saw_kubernetes.gate import projection + agent(source) + binding = source[1]["bindings"]["default/toolbox"] + binding["toolGate"] = { + "ownerLogin": "alice", + "trusted": {"claimName": "saw-gate-test", "uid": "gate-uid", "readOnly": True}, + "audit": {"claimName": "saw-audit-test", "uid": "audit-uid", "readOnly": False}, + } + plan = compile_profile(*source) + spec = plan["workspaces"][0]["sandboxes"][0]["spec"] + cm, _ = projection(source[1]["namespace"], "alice", source[1]["ownerSubject"], "default", "toolbox") + assert cm["metadata"]["annotations"]["saw.redhat.com/tool-gate-sha256"] in spec["command"][2] + mounts = spec["template"]["driver_config"]["kubernetes"]["containers"]["agent"]["volume_mounts"] + assert [(m["mount_path"], m["read_only"]) for m in mounts[1:]] == [ + ("/sandbox/.saw/trusted", True), ("/sandbox/.saw/audit", False)] + binding["toolGate"]["trusted"]["readOnly"] = False + with pytest.raises(ContractError, match="mount mode"): + compile_profile(*source) + + +def test_writer_key_binds_the_compiled_extension_and_tool_selection(source): + import base64 + from pathlib import Path + from saw_kubernetes.core import shared + + agent(source) + binding = source[1]["bindings"]["default/toolbox"] + # SubjectPublicKeyInfo prefix for Ed25519 followed by 32 fixture bytes. + key = base64.b64encode(bytes.fromhex("302a300506032b6570032100") + bytes(range(32))).decode() + binding["toolGate"] = {"ownerLogin":"alice", "receiptKey":key, + "trusted":{"claimName":"saw-gate-test","uid":"gate-uid","readOnly":True}, + "audit":{"claimName":"saw-audit-inbox","uid":"queue-uid","readOnly":False}} + plan = compile_profile(*source) + spec = plan["workspaces"][0]["sandboxes"][0]["spec"] + identity = shared.tool_gate_identity("alice", source[1]["ownerSubject"], "default", "toolbox") + tree, _, digest, _ = shared.tool_gate_bundle(Path(shared.__file__).parent, identity, key) + assert key.encode() in tree["plugin/index.js"][0] + assert digest in spec["command"][2] and "allow:['read','write','exec']" in spec["command"][2] + binding["toolGate"]["receiptKey"] = "invalid" + with pytest.raises(ContractError, match="Ed25519"): + compile_profile(*source) + + +@pytest.mark.parametrize("kind",["openclaw","nemoclaw"]) +def test_openai_model_uses_shared_bom_resolution(source,kind): + agent(source) + ref={"secretKeyRef":{"name":"inference","key":"custom_model"},"default":"gpt-4.1-mini"} + def configure(sb): + sb["type"]=kind + sb["inference"]["model"]=ref + sandbox(source,configure) + def model(): + return compile_profile(*source)["workspaces"][0]["sandboxes"][0]["spec"]["command"][-1] + assert model()=="gpt-4.1-mini" + source[1]["modelInputs"]={"inference.custom_model":"gpt-4.1"} + assert model()=="gpt-4.1" diff --git a/tests/kubernetes_runtime/test_portal_contract.py b/tests/kubernetes_runtime/test_portal_contract.py new file mode 100644 index 000000000..5b7eff77c --- /dev/null +++ b/tests/kubernetes_runtime/test_portal_contract.py @@ -0,0 +1,71 @@ +import copy +import subprocess +from functools import lru_cache + +import pytest +from saw_kubernetes.boundary import BoundaryError, objects +from saw_kubernetes.portal_contract import contract, verify_contract + + +@lru_cache(maxsize=1) +def public_test_ca(): + return subprocess.check_output(["openssl","req","-x509","-newkey","ec","-pkeyopt","ec_paramgen_curve:P-256","-nodes","-subj","/CN=saw-test","-keyout","/dev/null"],stderr=subprocess.DEVNULL,text=True) + + +def resources(reg="0123456789abcdef", owner="12345678-1234-4123-a123-123456789012", address="10.0.0.18"): + cfg=dict(namespace="runtime",trustedNamespace="saw-ctl-"+reg,ownerID=owner,governanceNamespace="gov",gateway="gw",proxyPolicy="proxy") + refs=objects(cfg) + data={p:{"metadata":{"uid":key},"spec":{},"data":{}} for key,(p,_) in refs.items()} + data[refs["gatewayRole"][0]]["rules"] = [] + def obj(k):return data[refs[k][0]] + obj("proxy")["spec"]={"namespace":cfg["trustedNamespace"],"owner":owner,"image":"locked"} + obj("gateway")["spec"]={"template":{"metadata":{"annotations":{"checksum/config":"c"*64}}}} + obj("gatewayService")["spec"]={"type":"ClusterIP","clusterIP":address,"clusterIPs":[address],"ipFamilies":["IPv4"],"ports":[{"port":17670}]} + pem=public_test_ca() + obj("publicCA").update(data={"ca.crt":pem},metadata={"labels":{"saw.redhat.com/runtime-owner":owner}}) + data["/api/v1/namespaces/"+cfg["trustedNamespace"]+"/configmaps/gw-proxy-ca"]={"metadata":{},"data":{"ca.crt":pem}} + return cfg,data,obj + + +def test_rebind_only_per_registration_identifiers(): + first,old,_=resources() + second,new,_=resources("fedcba9876543210","87654321-1234-4123-a123-123456789012","10.0.0.99") + verify_contract(second,new.__getitem__,contract(first,old.__getitem__)) + + +@pytest.mark.parametrize("change",["proxyImage","servicePort","governanceUID","fakeCA","replacementIssuer","extra"]) +def test_unreviewed_differences_rejected(change): + cfg,live,obj=resources() + expected=copy.deepcopy(contract(cfg,live.__getitem__)) + if change=="proxyImage":obj("proxy")["spec"]["image"]="foreign" + if change=="servicePort":obj("gatewayService")["spec"]["ports"][0]["port"]=80 + if change=="governanceUID":obj("governance")["metadata"]["uid"]="foreign" + if change=="fakeCA":obj("publicCA")["data"]["ca.crt"]="-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----" + if change=="replacementIssuer":obj("gatewayConfig")["data"]["issuer"]="foreign" + if change=="extra":expected["newKey"]="injected" + with pytest.raises(BoundaryError):verify_contract(cfg,live.__getitem__,expected) + + +@pytest.mark.parametrize("keys", [["w/a"], ["w/a", "w/b"]]) +def test_admission_rebinds_only_the_same_slot(keys): + import hashlib + + def fixture(registration): + config, live, obj = resources(registration) + config["slotKeys"] = keys + rules = [] + for slot in keys: + relay = hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] if len(keys) > 1 else registration + rules.append({"slot": slot, "proof": f"ui-{relay}-browser-proof", + "prep": f"gate-{relay}-" if len(keys) > 1 else "saw-gate-preparation-", + "claim": "gw-" + relay[:8] if len(keys) > 1 else "gw"}) + obj("validation")["spec"] = {"rules": rules} + return config, live, rules + + first, old, _ = fixture("0123456789abcdef") + second, new, rules = fixture("fedcba9876543210") + approved = contract(first, old.__getitem__) + verify_contract(second, new.__getitem__, approved) + rules[0]["proof"] = "ui-0123456789abcdef-browser-proof" + with pytest.raises(BoundaryError, match="validation"): + verify_contract(second, new.__getitem__, approved) diff --git a/tests/kubernetes_runtime/test_portal_profile.py b/tests/kubernetes_runtime/test_portal_profile.py new file mode 100644 index 000000000..d00003610 --- /dev/null +++ b/tests/kubernetes_runtime/test_portal_profile.py @@ -0,0 +1,54 @@ +import copy +from pathlib import Path +import re +import pytest +import yaml +from saw_kubernetes.compiler import compile_profile, ContractError + +ROOT = Path(__file__).resolve().parents[2] + + +def test_enabled_builtin_agent_profiles_pin_their_images(): + profiles = ROOT / "charts/saw-bom/profiles" + for path in profiles.rglob("sandbox.yaml"): + document = yaml.safe_load(path.read_text()) + for sandbox in document["spec"]["sandboxes"]: + if sandbox.get("enabled") and sandbox.get("type") in {"openclaw", "nemoclaw"}: + assert re.fullmatch(r"[^@\s]+@sha256:[0-9a-f]{64}", sandbox["image"]), path + + +def test_portal_bom_route_requires_same_sandboxes_browser_binding(): + profile=ROOT/"tests/fixtures/profiles/audited-openclaw" + cfg={"namespace":"saw-owner","namespaceUID":"ns","ownerID":"76c95ca5-b9bc-4be7-b764-bc028087ae28", + "ownerSubject":"owner","mode":"container","gatewayVersion":"0.1.2-rhaiv.6", + "approvedCredentialFreeProviders":[],"proxyBindings":[{"type":"openai", "secret":"inference", "key":"api_key", "refresh":False}],"bindings":{"agent/audited":{ + "workspace":{"claimName":"tenant-data","uid":"claim","readOnly":False}, + "toolGate":{"ownerLogin":"owner", "trusted":{"claimName":"saw-gate-fixture","uid":"gate","readOnly":True}, + "audit":{"claimName":"saw-audit-fixture","uid":"audit","readOnly":False}}, + "browser":{"origin":"https://owner.example.test","ownerLogin":"owner"}}}} + plan=compile_profile(profile,cfg) + assert plan["workspaces"][0]["sandboxes"][0]["name"]=="audited" + bad=copy.deepcopy(cfg) + del bad["bindings"]["agent/audited"]["browser"] + with pytest.raises(ContractError, match="UI route"): + compile_profile(profile,bad) + + +def test_receipt_key_validates_public_spki_and_service(): + from contextlib import closing + from io import BytesIO + from saw_kubernetes.receipt import receipt_key + import base64 + key=base64.b64encode(bytes.fromhex("302a300506032b6570032100")+"x".encode()*32) + class Writer: + data=key + def open(self,url,timeout): + assert url=="https://writer.archive.svc:8443/v1/receipt-key" + return closing(BytesIO(self.data)) + assert receipt_key("writer","archive",None,opener=Writer()) == key.decode() + w=Writer() + w.data=base64.b64encode(bytes(44)) + with pytest.raises(ValueError,match="Ed25519"): + receipt_key("writer","archive",None,opener=w) + with pytest.raises(ValueError,match="service"): + receipt_key("external.invalid/foo","archive",None,opener=w) diff --git a/tests/kubernetes_runtime/test_provider_governance.py b/tests/kubernetes_runtime/test_provider_governance.py new file mode 100644 index 000000000..7d78a1f99 --- /dev/null +++ b/tests/kubernetes_runtime/test_provider_governance.py @@ -0,0 +1,20 @@ +from saw_kubernetes import provider as governed + + +def test_brave_search_uses_shipped_governance_profile(): + slot, hosts = governed.requirements("brave") + + assert slot["name"] == "api_key" + assert slot["env_vars"] == ["BRAVE_API_KEY"] + assert hosts == ["api.search.brave.com"] + + credentials = {"BRAVE_API_KEY": "saw-cred.web-search.api_key"} + policy = { + "markerPrefix": "saw-cred", + "profiles": {"brave": hosts}, + "allowHosts": hosts, + "bindings": [{"providers": ["brave"], "secret": "web-search", "key": "api_key"}], + } + + assert governed.check_policy("brave", credentials, policy) + assert not governed.check_policy("brave", credentials, {**policy, "allowHosts": []}) diff --git a/tests/kubernetes_runtime/test_provider_recovery.py b/tests/kubernetes_runtime/test_provider_recovery.py new file mode 100644 index 000000000..9208df716 --- /dev/null +++ b/tests/kubernetes_runtime/test_provider_recovery.py @@ -0,0 +1,96 @@ +"""CreateProvider ambiguity must use a replay receipt, never redacted Get data.""" +import copy + +import pytest + +from saw_kubernetes import ContractError, Reconciler, RpcError, compile_profile +from .test_openclaw import agent +from .test_runtime import FakeGateway, source + + +class LostProviderResponse(FakeGateway): + """Model the pinned gateway's caller/request/payload/revision replay fence.""" + + def __init__(self): + super().__init__() + self.receipt = None + self.creations = 0 + self.replays = 0 + + def call(self, method, request): + if method != "CreateProvider": + return super().call(method, request) + ws = request["workspace_scope"]["workspace"] + key = (ws, request["provider"]["metadata"]["name"]) + if self.receipt: + original, response = self.receipt + assert request == original + self.replays += 1 + if self.providers[key] != response["provider"]: + raise RpcError("FAILED_PRECONDITION") + return copy.deepcopy(response) + if key in self.providers: + raise RpcError("ALREADY_EXISTS") + response = super().call(method, request) + response["provider"]["metadata"]["resource_version"] = "1" + response["provider"]["credentials"] = {k:"REDACTED" for k in request["provider"]["credentials"]} + self.receipt = copy.deepcopy((request, response)) + self.creations += 1 + raise RpcError("DEADLINE_EXCEEDED") + + +@pytest.fixture +def interrupted(source, tmp_path): + agent(source) + plan = compile_profile(*source) + ws = plan["workspaces"][0] + provider = ws["providers"][0] + gateway = LostProviderResponse() + core = Reconciler(gateway, tmp_path / "ledger", lambda _: None) + with pytest.raises(RpcError, match="DEADLINE_EXCEEDED"): + core._apply_provider(ws, provider, plan, {}, 1, []) + # Reconstruct the process from durable state; changing intent generation + # must not change the original CreateProvider request ID. + core = Reconciler(gateway, tmp_path / "ledger", lambda _: None) + return core, gateway, ws, provider, plan + + +def test_lost_provider_response_recovers_across_process_and_generation(interrupted): + core, gateway, ws, provider, plan = interrupted + state = core.store.load() + assert state["pendingProxyProviders"] and not state.get("proxyProviderRevisions") + assert core._apply_provider(ws, provider, plan, state, 2, []) + assert gateway.creations == 1 and gateway.replays == 1 + assert not core.store.load()["pendingProxyProviders"] + assert core._apply_provider(ws, provider, plan, core.store.load(), 2, []) + assert gateway.replays == 1 + + +@pytest.mark.parametrize("change", ["revision", "replacement", "foreign-owner"]) +def test_recovery_refuses_changed_gateway_resource(interrupted, change): + core, gateway, ws, provider, plan = interrupted + meta = gateway.providers[(ws["name"], provider["metadata"]["name"])]["metadata"] + if change == "revision": + meta["resource_version"] = "2" + elif change == "replacement": + meta["id"] = "replacement" + else: + meta["labels"] = {} + with pytest.raises(RpcError, match="FAILED_PRECONDITION"): + core._apply_provider(ws, provider, plan, core.store.load(), 2, []) + assert not core.store.load().get("proxyProviderRevisions") + + +def test_recovery_refuses_changed_marker_or_missing_receipt(interrupted): + core, gateway, ws, provider, plan = interrupted + changed = copy.deepcopy(provider) + changed["credentials"]["OPENAI_API_KEY"] = "saw-cred.other.api_key" + with pytest.raises(ContractError, match="pending provider creation changed"): + core._apply_provider(ws, changed, plan, core.store.load(), 2, []) + assert gateway.replays == 0 + gateway.receipt = None + with pytest.raises(ContractError, match="receipt unavailable"): + core._apply_provider(ws, provider, plan, core.store.load(), 2, []) + # Losing the local intent must not allow adoption based on labels alone. + with pytest.raises(ContractError, match="credential revision"): + core._apply_provider(ws, provider, plan, {}, 2, []) diff --git a/tests/kubernetes_runtime/test_proxy_ca_projection.py b/tests/kubernetes_runtime/test_proxy_ca_projection.py new file mode 100644 index 000000000..1efae4a43 --- /dev/null +++ b/tests/kubernetes_runtime/test_proxy_ca_projection.py @@ -0,0 +1,126 @@ +"""Public trust projection: no Secret reads, adoption, replacement or rotation.""" +import copy +import importlib.util +from pathlib import Path +import subprocess +import pytest + +ROOT=Path(__file__).resolve().parents[2] +SPEC=importlib.util.spec_from_file_location('project_ca',ROOT/'charts/saw-kubernetes-proxy/files/project_ca.py') +mod=importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(mod) + + +@pytest.fixture(scope='module') +def cert(tmp_path_factory): + root=tmp_path_factory.mktemp('proxy-ca-test') + subprocess.run(['openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','1', + '-subj','/CN=Public test CA','-keyout',str(root/'key.pem'),'-out',str(root/'ca.pem')], + check=True,capture_output=True) + value=(root/'ca.pem').read_text() + (root/'key.pem').unlink() + return value + + +@pytest.fixture +def config(): + return dict(ownerID='67a9891d-e281-442f-86c3-41771f5ce049',namespace='saw-test-trusted', + targetNamespace='saw-test',name='saw-test-proxy-ca') + + +class Api: + def __init__(self,config,cert): + self.objects={f'/api/v1/namespaces/{ns}':{'metadata':{'name':ns,'labels':{mod.OWNER:config['ownerID']}}} + for ns in [config['namespace'],config['targetNamespace']]} + self.source=f"/api/v1/namespaces/{config['namespace']}/configmaps/{config['name']}" + self.target=f"/api/v1/namespaces/{config['targetNamespace']}/configmaps/{config['name']}" + self.objects[self.source]={'metadata':{'name':config['name'],'namespace':config['namespace']},'data':{'ca.crt':cert}} + self.calls=[] + self.race=None + + def call(self,method,path,body=None): + self.calls.append((method,path)) + assert '/secrets' not in path + assert method in ['GET','POST'] + if method=='POST': + target=path+'/'+body['metadata']['name'] + if self.race: + body=self.race(copy.deepcopy(body)) + self.objects.setdefault(target,copy.deepcopy(body)) + return copy.deepcopy(self.objects.get(path)) + + +def test_repeat_sync_and_only_public_material(config,cert): + api=Api(config,cert) + mod.project(api,config) + before=copy.deepcopy(api.objects) + mod.project(api,config) + assert api.objects==before + assert sum(m=='POST' for m,p in api.calls)==1 + target=api.objects[api.target] + assert target['immutable'] is True and target['data']=={'ca.crt':cert} + assert target['metadata']['annotations']['argocd.argoproj.io/sync-options']==mod.KEEP + + +@pytest.mark.parametrize('mutation',[ + lambda b: b.update(immutable=False), + lambda b: b.update(data={'ca.crt':'different'}), + lambda b: b.update(binaryData={'private':'data'}), + lambda b: b['metadata'].update(ownerReferences=[{'name':'foreign'}]), + lambda b: b['metadata'].update(deletionTimestamp='now'), + lambda b: b['metadata'].update(labels={mod.OWNER:'foreign'}), + lambda b: b['metadata'].update(annotations={}), + lambda b: b.update(metadata=None), +]) +def test_refuses_foreign_changed_or_malformed_target(config,cert,mutation): + api=Api(config,cert) + mod.project(api,config) + mutation(api.objects[api.target]) + before=copy.deepcopy(api.objects) + with pytest.raises(mod.Failure):mod.project(api,config) + assert api.objects==before + + +def test_conflicting_create_race_is_not_adopted(config,cert): + api=Api(config,cert) + def race(body): + body['metadata']['labels'][mod.OWNER]='foreign' + return body + api.race=race + with pytest.raises(mod.Failure,match='explicit trust recovery'):mod.project(api,config) + + +def test_matching_create_race_is_safe(config,cert): + api=Api(config,cert) + api.race=lambda body:body + mod.project(api,config) + + +@pytest.mark.parametrize('value',['secret','-----BEGIN PRIVATE KEY-----','not a certificate',None]) +def test_invalid_source_rejected_before_any_write(config,cert,value): + api=Api(config,cert) + api.objects[api.source]['data']={'ca.crt':value} + with pytest.raises(mod.Failure):mod.project(api,config) + assert not any(m=='POST' for m,p in api.calls) + + +def test_extra_text_cannot_be_projected(config,cert): + api=Api(config,cert) + api.objects[api.source]['data']['ca.crt']=cert+'secret trailing text' + with pytest.raises(mod.Failure):mod.project(api,config) + + +def test_namespace_ownership_required_before_reading_ca(config,cert): + api=Api(config,cert) + api.objects['/api/v1/namespaces/saw-test']['metadata']['labels']={} + with pytest.raises(mod.Failure,match='ownership'):mod.project(api,config) + assert all('/configmaps' not in p for m,p in api.calls) + + +@pytest.mark.parametrize('patch',[{'ownerID':'0'*32},{'targetNamespace':'saw-test-trusted'}, + {'name':'../secrets'},{'targetNamespace':'kube-system'},{'extra':'field'}]) +def test_bad_config_never_calls_api(config,cert,patch): + api=Api(config,cert) + config.update(patch) + with pytest.raises(mod.Failure):mod.project(api,config) + assert not api.calls diff --git a/tests/kubernetes_runtime/test_relay_binding.py b/tests/kubernetes_runtime/test_relay_binding.py new file mode 100644 index 000000000..323d6bb08 --- /dev/null +++ b/tests/kubernetes_runtime/test_relay_binding.py @@ -0,0 +1,41 @@ +from pathlib import Path +import runpy +from unittest.mock import Mock +import uuid + +import pytest + +from saw_kubernetes.reconcile import RpcError + + +bind = runpy.run_path(str(Path(__file__).resolve().parents[2] / "charts/saw-workspace-relay/files/bind.py"))["bind"] +wid = str(uuid.UUID(int=1)) +meta = {"id": wid, "name": "default"} +admin = {"principal_subject": "owner", "role": "WORKSPACE_ROLE_ADMIN"} +reader = {"principal_subject": "relay", "role": "WORKSPACE_ROLE_USER"} + + +def run(responses): + gateway = Mock() + gateway.call.side_effect = responses + bind(gateway, "default", "", "relay", "owner", "registration", True, sleep=lambda _: None) + return [call.args[0] for call in gateway.call.call_args_list] + + +def test_controller_creates_workspace_before_assigning_owner(): + workspace = {"workspace": {"metadata": meta}} + calls = run([RpcError("NOT_FOUND"), workspace, {"members": []}, workspace, {"members": [admin]}, + {}, {"members": [admin, reader]}]) + assert calls.count("AddWorkspaceMember") == 1 + assert calls[-2:] == ["AddWorkspaceMember", "ListWorkspaceMembers"] + + +@pytest.mark.parametrize("responses, error", [ + ([{"workspace": {"metadata": meta}}, {}, {"workspace": {"metadata": {**meta, "id": str(uuid.UUID(int=2))}}}], + "workspace identity mismatch"), + ([{"workspace": {"metadata": meta}}, {"members": [{**reader, "principal_subject": "owner"}]}], "owner mismatch"), + ([{"workspace": {"metadata": meta}}, {"members": [{**admin, "principal_subject": "other"}]}], "unexpected membership"), +]) +def test_binding_denies_replacement_or_unexpected_roles(responses, error): + with pytest.raises(ValueError, match=error): + run(responses) diff --git a/tests/kubernetes_runtime/test_runtime.py b/tests/kubernetes_runtime/test_runtime.py new file mode 100644 index 000000000..53babadd9 --- /dev/null +++ b/tests/kubernetes_runtime/test_runtime.py @@ -0,0 +1,658 @@ +import copy +import json +import uuid + +import pytest +import yaml + +from saw_kubernetes import ContractError, Reconciler, RpcError, compile_profile, compile_profile_files +from saw_kubernetes.adapters import GrpcGateway, KubernetesResourceVerifier, OcReader +from saw_kubernetes.compiler import MANAGED, OWNER, canonical, digest +from saw_kubernetes.reconcile import Busy, StateStore + + +@pytest.fixture +def source(tmp_path): + root = tmp_path / "profile" + ws = root / "default" + ws.mkdir(parents=True) + docs = { + "workspace.yaml": {"apiVersion": "saw.redhat.com/v1alpha1", "kind": "Workspace", + "metadata": {"name": "default"}}, + "providers.yaml": {"apiVersion": "saw.redhat.com/v1alpha1", "kind": "Providers", + "spec": {"providers": []}}, + "sandbox.yaml": {"apiVersion": "saw.redhat.com/v1alpha1", "kind": "Sandboxes", + "spec": {"sandboxes": [{"name": "toolbox", "type": "generic", + "image": "registry.example/sandbox@sha256:" + "a" * 64, + "providers": []}]}}, + } + for name, content in docs.items(): + (ws / name).write_text(yaml.safe_dump(content)) + config = {"ownerID": "b032b2e4-2cd5-47a7-8ef8-ac73e15700b1", "namespace": "saw-alice", + "namespaceUID": "namespace-uid", "ownerSubject": "alice-sub", "mode": "container", + "gatewayVersion": "0.1.2-rhaiv.6", "approvedCredentialFreeProviders": ["tools"], + "bindings": {"default/toolbox": {"workspace": { + "claimName": "workspace-toolbox", "uid": "pvc-uid", "readOnly": False}}}} + return root, config + + +def mutate(source, filename, fn): + path = source[0] / "default" / filename + doc = yaml.safe_load(path.read_text()) + fn(doc) + path.write_text(yaml.safe_dump(doc)) + + +def sandbox(source, fn): + mutate(source, "sandbox.yaml", lambda d: fn(d["spec"]["sandboxes"][0])) + + +def test_compiler_is_deterministic_and_uses_stock_driver_envelope(source): + root, config = source + first = compile_profile(root, config) + assert first == compile_profile(root, json.loads(json.dumps(config, sort_keys=True))) + sb = first["workspaces"][0]["sandboxes"][0] + assert sb["spec"]["command"] == ["sh", "-c", "sleep infinity"] + driver = sb["spec"]["template"]["driver_config"]["kubernetes"] + assert driver["volumes"][0]["name"] == "saw-workspace" + assert driver["volumes"][0]["persistent_volume_claim"] == { + "claim_name": "workspace-toolbox", "read_only": False} + assert driver["containers"]["agent"]["volume_mounts"] == [ + {"name": "saw-workspace", "mount_path": "/sandbox/workspace", "read_only": False}] + assert first["claims"] == [{"name": "workspace-toolbox", "uid": "pvc-uid", "workspace": "default"}] + + +def test_mount_count_respects_governance_envelope(source): + binding = source[1]["bindings"]["default/toolbox"] + binding["mounts"] = [{"claimName": f"data-{i}", "uid": f"uid-{i}", + "readOnly": True, "mountPath": f"/data/model-{i}"} + for i in range(17)] + with pytest.raises(ContractError, match="at most 16"): + compile_profile(*source) + binding["mounts"].pop() + assert len(compile_profile(*source)["claims"]) == 17 + + +def flat_profile(source): + return {f"profiles__developer__default__{path.name}": path.read_text() + for path in (source[0] / "default").iterdir()} + + +def test_configmap_frontend_and_directory_frontend_have_identical_plans(source): + assert compile_profile_files(flat_profile(source), source[1]) == compile_profile(*source) + + +def test_profiles_cannot_claim_the_same_workspace_across_configmap_directories(source): + files = flat_profile(source) + files.update({key.replace("developer", "another"): value for key, value in list(files.items())}) + with pytest.raises(ContractError, match="shared SAW"): + compile_profile_files(files, source[1]) + + +@pytest.mark.parametrize("setting", [ + {"model": "model-that-would-be-ignored"}, {"agent": "custom-agent"}, + {"env": {"PROJECT": {"value": "example"}}}, + {"ui": {"route": False}}, {"harnessRequired": True}, +]) +def test_shared_valid_fields_still_require_backend_capability(source, setting): + sandbox(source, lambda sb: sb.update(setting)) + with pytest.raises(ContractError, match="capability"): + compile_profile(*source) + + +def test_invalid_yaml_does_not_echo_input_contents(source): + files = flat_profile(source) + files["profiles__developer__default__sandbox.yaml"] = "secret-input: [NEVER-ECHO," + with pytest.raises(ContractError) as exc: + compile_profile_files(files, source[1]) + assert "NEVER-ECHO" not in str(exc.value) + + +@pytest.mark.parametrize("kind", ["workspace", "sandbox", "provider"]) +def test_dns_name_must_match_the_whole_string(source, kind): + if kind == "workspace": + mutate(source, "workspace.yaml", lambda d: d["metadata"].update(name="default\n")) + elif kind == "sandbox": + sandbox(source, lambda sb: sb.update(name="toolbox\n")) + else: + mutate(source, "providers.yaml", lambda d: d["spec"]["providers"].append( + {"name": "tools\n", "type": "tools\n", "credentialFree": True})) + with pytest.raises(ContractError): + compile_profile(*source) + + +def test_profile_source_cannot_redirect_to_another_file(source, tmp_path): + path = source[0] / "default" / "sandbox.yaml" + moved = tmp_path / "outside.yaml" + path.rename(moved) + path.symlink_to(moved) + with pytest.raises(ContractError, match="symlinks"): + compile_profile(*source) + + +@pytest.mark.parametrize("extra", [ + {"type": "openclaw"}, {"type": "nemoclaw"}, {"harnessRef": {"name": "harness"}}, + {"env": {"TOKEN": {"secret": "inference"}}}, {"driver_config": {"hostPath": "/"}}, + {"serviceAccountName": "admin"}, {"ui": {"route": True}}, {"inference": {"providerRef": "nvidia"}}, + {"image": "registry.example/sandbox:latest"}, {"enabled": "false"}, {"gpu": {"enabled": True, "count": 1}}, +]) +def test_unported_or_privileged_profile_fails_before_rpc(source, extra): + sandbox(source, lambda sb: sb.update(extra)) + with pytest.raises(ContractError): + compile_profile(*source) + + +def test_credential_provider_not_silently_treated_as_keyless(source): + mutate(source, "providers.yaml", lambda d: d["spec"]["providers"].append( + {"name": "nvidia", "type": "nvidia", "credentialSecret": "inference"})) + with pytest.raises(ContractError): + compile_profile(*source) + + +def test_governed_credential_free_provider_compiles(source): + mutate(source, "providers.yaml", lambda d: d["spec"]["providers"].append( + {"name": "tools", "type": "tools", "credentialFree": True})) + sandbox(source, lambda sb: sb.update(providers=["tools"])) + plan = compile_profile(*source) + assert plan["workspaces"][0]["providers"][0]["type"] == "tools" + assert "credentials" not in canonical(plan) + + +@pytest.mark.parametrize("path", ["/", "/etc", "/sandbox/harness", "/data/../etc", "/data//files", "/data/", "/data/\x00"]) +def test_unapproved_mount_paths_rejected(source, path): + source[1]["bindings"]["default/toolbox"]["mounts"] = [ + {"claimName": "data", "uid": "data-uid", "readOnly": False, "mountPath": path}] + with pytest.raises(ContractError): + compile_profile(*source) + + +def test_rw_and_ro_mount_flags_both_reach_driver(source): + source[1]["bindings"]["default/toolbox"]["mounts"] = [ + {"claimName": "data", "uid": "data-uid", "readOnly": True, "mountPath": "/data"}] + driver = compile_profile(*source)["workspaces"][0]["sandboxes"][0]["spec"]["template"]["driver_config"]["kubernetes"] + assert driver["volumes"][1]["persistent_volume_claim"]["read_only"] is True + assert driver["containers"]["agent"]["volume_mounts"][1]["read_only"] is True + + +def test_native_gc_workspace_is_never_implicitly_created(source): + source[1]["bindings"] = {} + with pytest.raises(ContractError, match="retained workspace PVC"): + compile_profile(*source) + + +class FakeGateway: + """In-memory stock RPC semantics; no claim to be live runtime validation.""" + + def __init__(self): + self.calls = [] + self.workspaces = {"default": {"metadata": {"name": "default", "id": "ws-1"}, + "status": {"phase": "WORKSPACE_PHASE_ACTIVE"}}} + self.members, self.providers, self.sandboxes = {}, {}, {} + self.fail_after_create = False + self.sandbox_receipts = {} + + def call(self, method, request): + self.calls.append((method, copy.deepcopy(request))) + ws = request.get("workspace_scope", {}).get("workspace") + key = (ws, request.get("name")) + if method == "GetGatewayInfo": + return {"gateway_version": "0.1.2-rhaiv.6", "compute_drivers": [{"name": "kubernetes"}]} + if method == "GetProviderProfile": + return {"profile": {"id": request["id"], "source": "interceptor/governance"}} + if method == "GetWorkspace": + if request["name"] not in self.workspaces: + raise RpcError("NOT_FOUND") + return {"workspace": copy.deepcopy(self.workspaces[request["name"]])} + if method == "CreateWorkspace": + self.workspaces[request["name"]] = {"metadata": {"name": request["name"], "id": str(uuid.uuid4()), "labels": request["labels"]}, + "status": {"phase": "WORKSPACE_PHASE_ACTIVE"}} + return {"workspace": self.workspaces[request["name"]]} + if method == "ListWorkspaceMembers": + return {"members": self.members.get(ws, [])} + if method == "AddWorkspaceMember": + self.members.setdefault(ws, []).append({"principal_subject": request["principal_subject"], "role": request["role"]}) + return {} + if method == "GetProvider": + if key not in self.providers: + raise RpcError("NOT_FOUND") + return {"provider": copy.deepcopy(self.providers[key])} + if method == "CreateProvider": + obj = copy.deepcopy(request["provider"]) + obj["metadata"].update(workspace=ws, id=str(uuid.uuid4())) + self.providers[(ws, obj["metadata"]["name"])] = obj + return {"provider": obj} + if method == "GetSandbox": + if key not in self.sandboxes: + raise RpcError("NOT_FOUND") + return {"sandbox": copy.deepcopy(self.sandboxes[key])} + if method == "CreateSandbox": + rid = request["request_id"] + if rid in self.sandbox_receipts: + return {"sandbox": copy.deepcopy(self.sandbox_receipts[rid])} + obj = {"metadata": {"name": request["name"], "workspace": ws, "id": str(uuid.uuid4()), + "labels": request["labels"], "annotations": request["annotations"]}, + "spec": copy.deepcopy(request["spec"]), "status": {"phase": "SANDBOX_PHASE_PROVISIONING"}} + self.sandboxes[key] = obj + self.sandbox_receipts[rid] = copy.deepcopy(obj) + if self.fail_after_create: + self.fail_after_create = False + raise RpcError("DEADLINE_EXCEEDED") + return {"sandbox": obj} + if method in {"StartSandbox", "StopSandbox"}: + self.sandboxes[key]["status"]["phase"] = "SANDBOX_PHASE_STARTING" if method == "StartSandbox" else "SANDBOX_PHASE_STOPPING" + return {"sandbox": self.sandboxes[key]} + raise AssertionError(method) + + +@pytest.fixture +def runtime(source, tmp_path): + plan = compile_profile(*source) + gateway = FakeGateway() + checks = [] + verifier = lambda p: checks.append(p["revision"]) + reconciler = Reconciler(gateway, tmp_path / "state", verifier) + return plan, gateway, reconciler, checks + + +def ready(runtime): + plan, gateway, reconciler, _ = runtime + for _ in range(5): + result = reconciler.reconcile(plan, generation=1) + for sb in gateway.sandboxes.values(): + sb["status"]["phase"] = "SANDBOX_PHASE_READY" + if result["phase"] == "GenericReady": + return + raise AssertionError("did not converge") + + +def test_reconcile_is_idempotent_and_checks_readiness(runtime): + plan, gateway, reconciler, checks = runtime + ready(runtime) + before = [c for c in gateway.calls if c[0].startswith(("Create", "Add"))] + assert reconciler.reconcile(plan, generation=1)["phase"] == "GenericReady" + assert [c for c in gateway.calls if c[0].startswith(("Create", "Add"))] == before + assert len(checks) >= 4 + assert not any(c[0].startswith("Delete") for c in gateway.calls) + + +def test_ambiguous_create_and_process_restart_do_not_duplicate(runtime): + plan, gateway, reconciler, _ = runtime + reconciler.reconcile(plan, generation=1) # membership + gateway.fail_after_create = True + assert reconciler.reconcile(plan, generation=1)["phase"] == "Retry" + restarted = Reconciler(gateway, reconciler.store.directory, lambda p: None) + assert restarted.reconcile(plan, generation=1)["phase"] == "Pending" + calls = [r["request_id"] for method, r in gateway.calls if method == "CreateSandbox"] + assert len(calls) == 2 and len(set(calls)) == 1 and len(gateway.sandboxes) == 1 + + +def test_stop_immediately_after_create_uses_durable_receipt(runtime): + plan, gateway, reconciler, _ = runtime + while not gateway.sandboxes: + assert reconciler.reconcile(plan, generation=1)["phase"] == "Pending" + sb = gateway.sandboxes[("default", "toolbox")] + assert reconciler.store.load()["objectIDs"]["sandbox/default/toolbox"] == sb["metadata"]["id"] + sb["status"]["phase"] = "SANDBOX_PHASE_READY" + assert reconciler.reconcile(plan, generation=2, desired_state="Stopped")["phase"] == "Pending" + assert gateway.calls[-1][0] == "StopSandbox" + + +def test_intent_generations_make_repeated_stop_start_unambiguous(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + sb = gateway.sandboxes[("default", "toolbox")] + for generation, target in [(2, "Stopped"), (3, "Running"), (4, "Stopped"), (5, "Running")]: + assert reconciler.reconcile(plan, generation=generation, desired_state=target)["phase"] == "Pending" + assert reconciler.reconcile(plan, generation=generation, desired_state=target)["phase"] == "Pending" + sb["status"]["phase"] = "SANDBOX_PHASE_STOPPED" if target == "Stopped" else "SANDBOX_PHASE_READY" + assert reconciler.reconcile(plan, generation=generation, desired_state=target)["phase"] in {"Stopped", "GenericReady"} + mutations = [r["request_id"] for m, r in gateway.calls if m in {"StartSandbox", "StopSandbox"}] + assert len(mutations) == len(set(mutations)) == 4 + with pytest.raises(ContractError, match="stale"): + reconciler.reconcile(plan, generation=2, desired_state="Stopped") + + +def test_retained_identity_detects_same_name_replacement(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + gateway.sandboxes[("default", "toolbox")]["metadata"]["id"] = "replacement-id" + with pytest.raises(ContractError, match="same-name replacement"): + reconciler.reconcile(plan, generation=1) + + +def test_foreign_object_is_not_adopted(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + gateway.sandboxes[("default", "toolbox")]["metadata"]["labels"][OWNER] = "someone-else" + with pytest.raises(ContractError, match="another installation"): + reconciler.reconcile(plan, generation=1) + + +def test_resource_verification_failure_prevents_all_rpc(runtime): + plan, gateway, reconciler, _ = runtime + def revoked(_): + raise ContractError("approval revoked") + reconciler.verify_resources = revoked + with pytest.raises(ContractError, match="revoked"): + reconciler.reconcile(plan, generation=1) + assert not gateway.calls + + +def test_lock_prevents_second_reconciler(runtime): + plan, _, reconciler, _ = runtime + with reconciler.store.locked(): + with pytest.raises(Busy): + Reconciler(reconciler.rpc, reconciler.store.directory, lambda p: None).reconcile(plan, generation=1) + + +def test_tampered_plan_and_owner_changes_rejected(runtime): + plan, _, reconciler, _ = runtime + ready(runtime) + changed = copy.deepcopy(plan) + changed["ownerSubject"] = "mallory" + with pytest.raises(ContractError, match="modified compiled plan"): + reconciler.reconcile(changed, generation=2) + changed["revision"] = digest({k: v for k, v in changed.items() if k != "revision"}) + with pytest.raises(ContractError, match="immutable"): + reconciler.reconcile(changed, generation=2) + + +def test_stopped_new_intent_does_not_briefly_start_workload(runtime): + plan, gateway, reconciler, _ = runtime + for _ in range(3): + assert reconciler.reconcile(plan, generation=1, desired_state="Stopped")["phase"] == "Stopped" + assert not gateway.sandboxes + assert not any(method.startswith(("Create", "Add", "Start", "Stop")) for method, _ in gateway.calls) + + +def test_shutdown_after_attachment_revocation_only_stops_owned_objects(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + def revoked(_): + raise ContractError("attachment revoked") + checked = [] + reconciler.verify_resources = revoked + reconciler.verify_shutdown = lambda p: checked.append(p["namespaceUID"]) + gateway.calls.clear() + reconciler.reconcile(plan, generation=2, desired_state="Stopped") + assert checked == [plan["namespaceUID"]] + assert {m for m, _ in gateway.calls} <= {"GetGatewayInfo", "GetWorkspace", "GetSandbox", "StopSandbox"} + assert any(m == "StopSandbox" for m, _ in gateway.calls) + with pytest.raises(ContractError, match="revoked"): + reconciler.reconcile(plan, generation=3, desired_state="Running") + + +@pytest.mark.parametrize("object_type", ["workspace", "sandbox"]) +def test_shutdown_rejects_same_name_replacement(runtime, object_type): + plan, gateway, reconciler, _ = runtime + ready(runtime) + if object_type == "workspace": + gateway.workspaces["default"]["metadata"]["id"] = "foreign-id" + else: + gateway.sandboxes[("default", "toolbox")]["metadata"]["id"] = "foreign-id" + gateway.calls.clear() + with pytest.raises(ContractError, match="replaced"): + reconciler.reconcile(plan, generation=2, desired_state="Stopped") + assert not any(m == "StopSandbox" for m, _ in gateway.calls) + + +def test_shutdown_without_recorded_ids_does_not_adopt_existing_sandbox(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + state = reconciler.store.load() + state["objectIDs"] = {} + reconciler.store.save(state) + gateway.calls.clear() + with pytest.raises(ContractError, match="unrecorded"): + reconciler.reconcile(plan, generation=2, desired_state="Stopped") + assert not any(m == "StopSandbox" for m, _ in gateway.calls) + + +def test_shutdown_also_covers_retained_objects_removed_from_desired_membership(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + removed = copy.deepcopy(plan) + removed["workspaces"][0]["sandboxes"] = [] + removed["revision"] = digest({k: v for k, v in removed.items() if k != "revision"}) + gateway.calls.clear() + reconciler.reconcile(removed, generation=2, desired_state="Stopped") + assert any(m == "StopSandbox" and r["name"] == "toolbox" for m, r in gateway.calls) + + +def test_shutdown_does_not_skip_its_identity_verifier(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + def replaced(_): + raise ContractError("namespace replaced") + reconciler.verify_shutdown = replaced + gateway.calls.clear() + with pytest.raises(ContractError, match="namespace replaced"): + reconciler.reconcile(plan, generation=2, desired_state="Stopped") + assert not gateway.calls + + +@pytest.mark.parametrize("mode", ["confidential", "vm"]) +def test_unsupported_backend_is_rejected_before_gateway_calls(source, mode): + source[1]["mode"] = mode + with pytest.raises(ContractError, match="only accepts container"): + compile_profile(*source) + + +@pytest.fixture +def resources(runtime): + plan = runtime[0] + ns = {"metadata": {"uid": plan["namespaceUID"], "labels": {OWNER: plan["ownerID"]}}} + pvc = {"metadata": {"uid": "pvc-uid", "labels": {OWNER: plan["ownerID"], + "openshell.ai/sandbox-attachable": "true", "openshell.ai/sandbox-attachable-workspace": "default"}, + "annotations": {"argocd.argoproj.io/sync-options": "Prune=false,Delete=false"}}, + "spec": {"volumeMode": "Filesystem"}, "status": {"phase": "Pending"}} + read = lambda kind, name, namespace=None: ns if kind == "namespace" else pvc + return plan, ns, pvc, KubernetesResourceVerifier(read, lambda plan: None) + + +def test_pending_wffc_claim_is_admitted_without_deadlock(resources): + plan, _, _, verifier = resources + verifier(plan) + + +def test_shutdown_verifier_checks_namespace_without_reattaching_revoked_storage(resources): + plan, ns, pvc, verifier = resources + pvc["metadata"]["labels"]["openshell.ai/sandbox-attachable"] = "false" + verifier.verify_boundary = lambda _: (_ for _ in ()).throw(ContractError("policy unavailable")) + verifier.verify_shutdown(plan) + ns["metadata"]["uid"] = "foreign" + with pytest.raises(ContractError, match="replaced"): + verifier.verify_shutdown(plan) + + +@pytest.mark.parametrize("mutation", [ + lambda p: p["metadata"].update(uid="replacement"), + lambda p: p["metadata"].update(deletionTimestamp="now"), + lambda p: p["metadata"].update(ownerReferences=[{"kind": "Sandbox"}]), + lambda p: p["metadata"].update(annotations={}), + lambda p: p["metadata"]["labels"].update({"openshell.ai/sandbox-attachable-workspace": "other"}), + lambda p: p["spec"].update(volumeMode="Block"), +]) +def test_replaced_unapproved_or_gc_owned_claim_refused(resources, mutation): + plan, _, pvc, verifier = resources + mutation(pvc) + with pytest.raises(ContractError): + verifier(plan) + + +def test_oc_reader_does_not_allow_secret_or_arbitrary_resource_access(): + with pytest.raises(ContractError, match="allowlist"): + OcReader()("secret", "inference") + + +def test_provider_preflight_occurs_before_mutation(source, tmp_path): + mutate(source, "providers.yaml", lambda d: d["spec"]["providers"].append( + {"name": "tools", "type": "tools", "credentialFree": True})) + sandbox(source, lambda sb: sb.update(providers=["tools"])) + plan = compile_profile(*source) + gateway = FakeGateway() + original = gateway.call + def request(method, body): + if method == "GetProviderProfile": + return {"profile": {"id": "tools", "source": "interceptor/governance", "credentials": [{"name": "key"}]}} + return original(method, body) + gateway.call = request + with pytest.raises(ContractError, match="credential-free"): + Reconciler(gateway, tmp_path / "state", lambda p: None).reconcile(plan, generation=1) + assert not any(m.startswith(("Create", "Add")) for m, _ in gateway.calls) + + +def test_disappeared_sandbox_does_not_recreate_or_erase_workspace(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + del gateway.sandboxes[("default", "toolbox")] + before = len([m for m, _ in gateway.calls if m == "CreateSandbox"]) + with pytest.raises(ContractError, match="refuse implicit recreation"): + reconciler.reconcile(plan, generation=1) + assert len([m for m, _ in gateway.calls if m == "CreateSandbox"]) == before + + +def test_spec_drift_never_triggers_delete_and_recreate(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + gateway.sandboxes[("default", "toolbox")]["spec"]["template"]["image"] = "unexpected" + with pytest.raises(ContractError, match="migration"): + reconciler.reconcile(plan, generation=1) + assert not any(m.startswith("Delete") for m, _ in gateway.calls) + + +@pytest.mark.parametrize("extra", [ + {"environment": {"UNAPPROVED": "value"}}, + {"resource_requirements": {"gpu": {"count": 2}}}, + {"resource_requirements": {"gpu": {}}}, # Presence requests the default GPU count. + {"log_level": "debug"}, + {"restart_policy": "SANDBOX_RESTART_POLICY_ALWAYS"}, + {"restart_policy": "SANDBOX_RESTART_POLICY_ON_FAILURE"}, + {"host_config": {"privileged": True}}, + {"future_unreviewed_field": "value"}, +]) +def test_unsupported_observed_spec_never_reports_ready_or_mutates(runtime, extra): + plan, gateway, reconciler, _ = runtime + ready(runtime) + gateway.sandboxes[("default", "toolbox")]["spec"].update(extra) + before = len(gateway.calls) + with pytest.raises(ContractError, match="configuration drift"): + reconciler.reconcile(plan, generation=1) + assert {method for method, _ in gateway.calls[before:]} <= { + "GetGatewayInfo", "GetWorkspace", "ListWorkspaceMembers", "GetSandbox"} + + +def test_gateway_defaults_and_governed_policy_are_not_user_spec_drift(runtime): + plan, gateway, reconciler, _ = runtime + ready(runtime) + spec = gateway.sandboxes[("default", "toolbox")]["spec"] + spec.update(restart_policy="SANDBOX_RESTART_POLICY_NEVER", + provider_attachment_epoch=str(uuid.uuid4()), + policy={"version": 1}, environment={}, resource_requirements={}, log_level="") + assert reconciler.reconcile(plan, generation=1)["phase"] == "GenericReady" + # Provider attachment epochs are gateway-owned and can rotate independently. + spec["provider_attachment_epoch"] = str(uuid.uuid4()) + assert reconciler.reconcile(plan, generation=1)["phase"] == "GenericReady" + + +@pytest.mark.parametrize("options", ["Prune=false", "Delete=false", "Prune=true,Delete=false", + "Prune=false,Delete=false,Delete=true", + "Prune=false,Delete=false,Prune=true"]) +def test_retained_pvc_requires_prune_and_application_deletion_protection(resources, options): + plan, _, pvc, verifier = resources + pvc["metadata"]["annotations"]["argocd.argoproj.io/sync-options"] = options + with pytest.raises(ContractError, match="prune and application-deletion protection"): + verifier(plan) + + +def test_retained_pvc_options_allow_order_and_whitespace(resources): + plan, _, pvc, verifier = resources + pvc["metadata"]["annotations"]["argocd.argoproj.io/sync-options"] = "Delete=false, Prune=false" + verifier(plan) + + +def test_transient_retries_have_bounded_backoff(runtime): + plan, gateway, reconciler, _ = runtime + def unavailable(*_): + raise RpcError("UNAVAILABLE") + gateway.call = unavailable + delays = [reconciler.reconcile(plan, generation=1)["retry_after_seconds"] for _ in range(9)] + assert delays == [2, 4, 8, 16, 32, 60, 60, 60, 60] + + +@pytest.mark.parametrize("endpoint", ["http://gateway:17670", "https://user:password@gateway", + "https://gateway/path", "https://gateway?token=secret", "https://gateway:bad", + "https://gateway:0", "https://gateway:\n443"]) +def test_invalid_or_insecure_grpc_endpoint_fails_before_identity_read(endpoint): + with pytest.raises(ContractError, match="gateway endpoint"): + GrpcGateway(endpoint, "/does-not-exist", "/does-not-exist", "/does-not-exist") + + +def test_lost_sandbox_reply_cannot_create_on_stop_or_adopt_a_replacement(runtime): + from saw_kubernetes.reconcile import MissingSandboxReceipt + plan, gateway, core, _ = runtime + core.reconcile(plan, generation=1) + gateway.fail_after_create = True + assert core.reconcile(plan, generation=1)["phase"] == "Retry" + with pytest.raises(MissingSandboxReceipt): + core.reconcile(plan, generation=2, desired_state="Stopped") + assert sum(m == "CreateSandbox" for m, _ in gateway.calls) == 1 + gateway.sandboxes[("default", "toolbox")]["metadata"]["id"] = "replacement" + with pytest.raises(ContractError, match="replacement"): + core.reconcile(plan, generation=3) + assert "sandbox/default/toolbox" not in core.store.load()["objectIDs"] + + +def test_frontend_attachment_check_preserves_standalone_boundary_requirement(): + reads = [] + boundary_calls = [] + plan = {"namespace":"saw-one", "namespaceUID":"uid", "ownerID":"owner", "claims":[], "runtimeClass":{"name":""}} + def read(kind, name, namespace=None): + reads.append((kind, name)) + return {"metadata":{"uid":"uid","labels":{OWNER:"owner"}}} + verifier = KubernetesResourceVerifier(read, lambda p: boundary_calls.append(p)) + verifier.verify_attachments(plan) + assert reads == [("namespace","saw-one")] and not boundary_calls + verifier(plan) + assert len(reads) == 2 and boundary_calls == [plan] + + +def test_kata_reuses_container_workload_requests(source): + profile, config = source + ordinary = compile_profile(profile, config) + config.update(mode="kata", workloadRuntimeClass="saw-kata", + workloadRuntimeClassUID=str(uuid.uuid4()), workloadRuntimeClassDigest="a" * 64) + plan = compile_profile(profile, config) + assert plan["workspaces"] == ordinary["workspaces"] + assert plan["requestedPlacement"] == {"supervisor": "container", "workload": "kata"} + assert plan["runtimeClass"]["uid"] == config["workloadRuntimeClassUID"] + assert ordinary["runtimeClass"] == {"name": "", "uid": ""} + + +@pytest.mark.parametrize("bad", [ + {"workloadRuntimeClass": ""}, {"workloadRuntimeClassUID": ""}, + {"workloadRuntimeClassUID": str(uuid.UUID(int=0))}, {"workloadRuntimeClassDigest": "bad"}, +]) +def test_kata_rejects_incomplete_review(source, bad): + profile, config = source + config.update(mode="kata", workloadRuntimeClass="saw-kata", + workloadRuntimeClassUID=str(uuid.uuid4()), workloadRuntimeClassDigest="a" * 64) + config.update(bad) + with pytest.raises(ContractError): + compile_profile(profile, config) + + +def test_live_attachment_verifier_checks_runtimeclass_drift(resources): + plan, namespace, pvc, _ = resources + placement = {"handler": "kata", "overhead": {"podFixed": {"memory": "256Mi"}}, + "scheduling": {"nodeSelector": {"pool": "dedicated"}}} + runtime = dict(placement, metadata={"uid": "reviewed", "labels": {"openshell.ai/sandbox-attachable": "true"}}) + plan["runtimeClass"] = {"name": "saw-kata", "uid": "reviewed", "sha256": digest(placement)} + objects = {"namespace": namespace, "persistentvolumeclaim": pvc, "runtimeclass": runtime} + verifier = KubernetesResourceVerifier(lambda kind, *args: objects[kind], lambda _: None) + verifier(plan) + for change in ({"handler": "runc"}, {"overhead": {}}, {"scheduling": {}}, + {"metadata": dict(runtime["metadata"], uid="replaced")}): + objects["runtimeclass"] = dict(runtime, **change) + with pytest.raises(ContractError): + verifier(plan) diff --git a/tests/kubernetes_runtime/test_shared_core.py b/tests/kubernetes_runtime/test_shared_core.py new file mode 100644 index 000000000..384875fb1 --- /dev/null +++ b/tests/kubernetes_runtime/test_shared_core.py @@ -0,0 +1,20 @@ +"""Cross-backend identity belongs to the suite that ships both adapters.""" +import importlib.util +from pathlib import Path +import sys + +from saw_kubernetes.core import shared, admin + + +def test_vm_and_kubernetes_load_the_same_canonical_module(monkeypatch): + root = Path(__file__).resolve().parents[2] + path = root / "charts/openshell-saw/files/installer/apply_bom.py" + spec = importlib.util.spec_from_file_location("test_saw_vm_installer", path) + installer = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, installer) + spec.loader.exec_module(installer) + assert shared is installer._saw_core + assert shared.Profile is installer.Profile + assert shared.InstallerError is installer.InstallerError + + assert admin is installer._saw_admin diff --git a/tests/kubernetes_runtime/test_stock_protobuf.py b/tests/kubernetes_runtime/test_stock_protobuf.py new file mode 100644 index 000000000..0584d2944 --- /dev/null +++ b/tests/kubernetes_runtime/test_stock_protobuf.py @@ -0,0 +1,201 @@ +"""Real pinned SDK serialization with an in-memory gRPC channel, no sockets. + +Run via kubernetes-runtime/test-stock-protobuf.py; ordinary unit runs skip this +module when the generated stock SDK has not been provisioned. +""" + +import copy +import os + +import pytest + +if os.environ.get("SAW_STOCK_SDK_TESTS") != "1": + pytest.skip("run test-stock-protobuf.py for pinned SDK wire-contract coverage", allow_module_level=True) + +import grpc +from google.protobuf.json_format import MessageToDict, ParseDict +from google.protobuf.message_factory import GetMessageClass +from openshell._proto import openshell_pb2 + +from saw_kubernetes import ContractError, Reconciler, RpcError, compile_profile +from saw_kubernetes.adapters import GrpcGateway, RPC_METHODS +from .test_runtime import FakeGateway, mutate, sandbox, source + + +class WireError(grpc.RpcError): + def __init__(self, name): + self.name = name + + def code(self): + return getattr(grpc.StatusCode, self.name) + + def details(self): + return "sensitive arbitrary server detail must not escape" + + +class WireChannel: + """Execute generated stub serializers and real request/response descriptors.""" + + def __init__(self, backend): + self.backend, self.calls, self.closed = backend, [], False + + def unary_unary(self, path, request_serializer, response_deserializer, **kwargs): + name = path.rsplit("/", 1)[1] + descriptor = openshell_pb2.DESCRIPTOR.services_by_name["OpenShell"].methods_by_name[name] + request_type = GetMessageClass(descriptor.input_type) + response_type = GetMessageClass(descriptor.output_type) + + def invoke(request, timeout): + serialized = request_serializer(request) + body = MessageToDict(request_type.FromString(serialized), preserving_proto_field_name=True) + self.calls.append((name, copy.deepcopy(body), timeout, serialized)) + try: + response = self.backend.call(name, body) + except RpcError as exc: + raise WireError(exc.code) from None + message = ParseDict(response, response_type(), ignore_unknown_fields=False) + return response_deserializer(message.SerializeToString()) + + return invoke + + # The generated OpenShell service also defines streams; this reconciler's + # allowlist does not use them, so attempts fail instead of silently stubbing. + def unary_stream(self, *args, **kwargs): + return lambda *_args, **_kwargs: pytest.fail("stream outside reconciler surface") + + stream_stream = unary_stream + stream_unary = unary_stream + + def close(self): + self.closed = True + + +@pytest.fixture +def transport(tmp_path, monkeypatch): + backend = FakeGateway() + channel = WireChannel(backend) + supplied = {} + + def credentials(roots, key, certificate): + supplied.update(roots=roots, key=key, certificate=certificate) + return "mock-mtls-credentials" + + def connect(target, identity): + supplied.update(target=target, identity=identity) + return channel + + monkeypatch.setattr(grpc, "ssl_channel_credentials", credentials) + monkeypatch.setattr(grpc, "secure_channel", connect) + for filename, value in [("ca", "root"), ("cert", "client"), ("key", "private")]: + (tmp_path / filename).write_text(value) + adapter = GrpcGateway("https://gateway.saw.svc:17670", tmp_path / "ca", tmp_path / "cert", + tmp_path / "key", timeout=12) + return adapter, backend, channel, supplied + + +def test_all_allowed_requests_are_stock_protobuf_methods(): + methods = openshell_pb2.DESCRIPTOR.services_by_name["OpenShell"].methods_by_name + for name in RPC_METHODS: + assert name in methods + assert not methods[name].client_streaming and not methods[name].server_streaming + assert getattr(openshell_pb2, name + "Request").DESCRIPTOR == methods[name].input_type + + +def test_generated_stub_uses_tls_inputs_and_deadline(transport): + adapter, _, channel, supplied = transport + result = adapter.call("GetGatewayInfo", {}) + assert result["gateway_version"] == "0.1.2-rhaiv.6" + assert supplied == {"roots": b"root", "key": b"private", "certificate": b"client", + "target": "gateway.saw.svc:17670", "identity": "mock-mtls-credentials"} + assert channel.calls[-1][:3] == ("GetGatewayInfo", {}, 12) + adapter.close() + assert channel.closed + + +def test_unknown_field_rejected_before_sending_and_errors_are_sanitized(transport): + adapter, _, channel, _ = transport + with pytest.raises(ContractError, match="protobuf API"): + adapter.call("GetWorkspace", {"name": "default", "secret": "do-not-send"}) + assert channel.calls == [] + with pytest.raises(RpcError) as error: + adapter.call("GetWorkspace", {"name": "absent"}) + assert str(error.value) == "NOT_FOUND" + assert "sensitive" not in str(error.value) + + +@pytest.mark.parametrize("workspace", ["default", "research"]) +def test_create_observe_stop_restart_roundtrips_real_stock_messages(source, transport, tmp_path, workspace): + mutate(source, "providers.yaml", lambda d: d["spec"]["providers"].append( + {"name": "tools", "type": "tools", "credentialFree": True})) + sandbox(source, lambda sb: sb.update(providers=["tools"])) + if workspace != "default": + mutate(source, "workspace.yaml", lambda d: d["metadata"].update(name=workspace)) + (source[0] / "default").rename(source[0] / workspace) + source[1]["bindings"][f"{workspace}/toolbox"] = source[1]["bindings"].pop("default/toolbox") + plan = compile_profile(*source) + adapter, backend, channel, _ = transport + reconciler = Reconciler(adapter, tmp_path / "state", lambda _: None) + for generation, desired, expected in [(1, "Running", "GenericReady"), (2, "Stopped", "Stopped"), + (3, "Running", "GenericReady")]: + for _ in range(12): + status = reconciler.reconcile(plan, generation=generation, desired_state=desired) + if status["phase"] == expected: + break + # Explicit fake driver progress, separate from gateway acceptance. + for instance in backend.sandboxes.values(): + phase = instance["status"]["phase"] + if phase in {"SANDBOX_PHASE_PROVISIONING", "SANDBOX_PHASE_STARTING"}: + instance["status"]["phase"] = "SANDBOX_PHASE_READY" + elif phase == "SANDBOX_PHASE_STOPPING": + instance["status"]["phase"] = "SANDBOX_PHASE_STOPPED" + assert status["phase"] == expected + names = {name for name, *_ in channel.calls} + assert {"CreateProvider", "CreateSandbox", "AddWorkspaceMember", "StartSandbox", "StopSandbox"} <= names + assert ("CreateWorkspace" in names) == (workspace != "default") + create = next(body for name, body, *_ in channel.calls if name == "CreateSandbox") + assert create["spec"]["template"]["driver_config"]["kubernetes"]["volumes"][0][ + "persistent_volume_claim"]["read_only"] is False + # Mutation IDs survive exact wire serialization, including retries. + assert len(create["request_id"]) == 36 + assert backend.sandboxes[(workspace, "toolbox")]["status"]["phase"] == "SANDBOX_PHASE_READY" + + +def test_wire_deadline_after_create_is_observed_after_reconciler_restart(source, transport, tmp_path): + plan = compile_profile(*source) + adapter, backend, channel, _ = transport + reconciler = Reconciler(adapter, tmp_path / "state", lambda _: None) + reconciler.reconcile(plan, generation=1) + backend.fail_after_create = True + status = reconciler.reconcile(plan, generation=1) + assert status["phase"] == "Retry" and status["reason"] == "DEADLINE_EXCEEDED" + original_id = backend.sandboxes[("default", "toolbox")]["metadata"]["id"] + restarted = Reconciler(adapter, tmp_path / "state", lambda _: None) + assert restarted.reconcile(plan, generation=1)["phase"] == "Pending" + creates = [body for name, body, *_ in channel.calls if name == "CreateSandbox"] + assert len(creates) == 2 and creates[0]["request_id"] and creates[0] == creates[1] + assert list(backend.sandbox_receipts) == [creates[0]["request_id"]] + assert list(backend.sandboxes) == [("default", "toolbox")] + assert backend.sandboxes[("default", "toolbox")]["metadata"]["id"] == original_id + assert restarted.store.load()["objectIDs"]["sandbox/default/toolbox"] == original_id + + +@pytest.mark.parametrize("extra", [ + {"environment": {"UNAPPROVED": "value"}}, + {"resource_requirements": {"gpu": {}}}, + {"restart_policy": "SANDBOX_RESTART_POLICY_ALWAYS"}, +]) +def test_wire_normalized_defaults_accepted_but_unsupported_spec_drift_rejected(source, transport, tmp_path, extra): + plan = compile_profile(*source) + adapter, backend, _, _ = transport + reconciler = Reconciler(adapter, tmp_path / "state", lambda _: None) + reconciler.reconcile(plan, generation=1) # membership + reconciler.reconcile(plan, generation=1) # create + instance = backend.sandboxes[("default", "toolbox")] + instance["status"]["phase"] = "SANDBOX_PHASE_READY" + instance["spec"].update(restart_policy="SANDBOX_RESTART_POLICY_NEVER", + provider_attachment_epoch="00000000-0000-4000-8000-000000000001", + policy={"version": 1}) + assert reconciler.reconcile(plan, generation=1)["phase"] == "GenericReady" + instance["spec"].update(extra) + with pytest.raises(ContractError, match="configuration drift"): + reconciler.reconcile(plan, generation=1) diff --git a/tests/openclaw-config-publication.mjs b/tests/openclaw-config-publication.mjs new file mode 100644 index 000000000..bd7d2c919 --- /dev/null +++ b/tests/openclaw-config-publication.mjs @@ -0,0 +1,71 @@ +// Run against a stock OpenClaw binary. No provider requests or real credentials. +// All config, backups and state live in one temporary directory and are removed. +import assert from 'node:assert/strict'; +import {mkdtempSync, readFileSync, writeFileSync, renameSync, rmSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const directory = mkdtempSync(join(tmpdir(), 'saw-config-publication-')); +const target = join(directory, 'openclaw.json'); +const batch = join(directory, 'managed.json'); +const executable = process.env.SAW_TEST_OPENCLAW || 'openclaw'; +const env = { + PATH: process.env.PATH, HOME: directory, OPENCLAW_HOME: directory, + OPENCLAW_STATE_DIR: directory, OPENCLAW_CONFIG_PATH: target, + XDG_CONFIG_HOME: directory, TMPDIR: directory, + OPENAI_API_KEY: 'synthetic-config-test-only' +}; +function cli(...args) { + const result = spawnSync(executable, args, {env, encoding:'utf8', timeout:120000}); + assert.equal(result.status, 0, `stock CLI failed at ${args.slice(0, 2).join(' ')} (${result.status}): ${(result.stderr || '').slice(-1200)}`); + return result.stdout.trim(); +} +function configuration(model, stale = false) { + const provider = {baseUrl:'https://api.openai.com/v1', api:'openai-completions', + apiKey:'${OPENAI_API_KEY}', models:[{id:model, name:model}]}; + return { + gateway:{mode:'local', bind:'loopback', port:19001}, + models:{providers:{'saw-openai':provider, ...(stale ? {'saw-stale':provider} : {})}}, + agents:{defaults:{model:{primary:'saw-openai/' + model}, workspace:join(directory, 'workspace')}}, + tools:stale ? {allow:['read']} : {deny:['*']}, + plugins:{allow:[], load:{paths:[]}, entries:{}}, skills:{load:{extraDirs:[]}}, + channels:{}, mcp:{servers:{}} + }; +} +function publish(value) { + writeFileSync(batch, JSON.stringify(Object.entries(value).map(([path, value]) => ({path, value}))), {mode:0o600}); + cli('config', 'set', '--batch-file', batch); +} +try { + publish(configuration('gpt-4.1-mini', true)); + // A second native write establishes a backup and observed baseline. + cli('config', 'set', 'gateway.port', '19002', '--strict-json'); + assert.equal(JSON.parse(cli('config', 'get', 'agents.defaults.model.primary', '--json')), 'saw-openai/gpt-4.1-mini'); + assert.ok(JSON.parse(readFileSync(target)).meta); + cli('plugins', 'list', '--json'); + + // Reproduce the old integration failure: atomically replacing the entire + // config without stock metadata causes recovery of the previous choice. + writeFileSync(target + '.replacement', JSON.stringify(configuration('gpt-4.1')), {mode:0o600}); + renameSync(target + '.replacement', target); + cli('plugins', 'list', '--json'); + const afterReplacement = JSON.parse(cli('config', 'get', 'agents.defaults.model.primary', '--json')); + assert.ok(['saw-openai/gpt-4.1-mini', 'saw-openai/gpt-4.1'].includes(afterReplacement)); + const legacyReplacementRestored = afterReplacement === 'saw-openai/gpt-4.1-mini'; + if (process.argv.includes('--expect-restore')) assert.ok(legacyReplacementRestored); + + publish(configuration('gpt-4.1-mini', true)); + publish(configuration('gpt-4.1')); + assert.equal(JSON.parse(cli('config', 'get', 'agents.defaults.model.primary', '--json')), 'saw-openai/gpt-4.1'); + const current = JSON.parse(readFileSync(target)); + assert.ok(current.meta); + assert.deepEqual(Object.keys(current.models.providers), ['saw-openai']); + assert.deepEqual(current.models.providers['saw-openai'].models.map(m => m.id), ['gpt-4.1']); + assert.equal(current.models.providers['saw-openai'].apiKey, '${OPENAI_API_KEY}'); + assert.deepEqual(current.tools, {deny:['*']}); + console.log(JSON.stringify({legacyReplacementRestored, nativePublicationRetainedExactModel:true, + metadataPreserved:true, credentialReferencePreserved:true, staleManagedFieldsRemoved:true, providerRequests:0})); +} finally { + rmSync(directory, {recursive:true, force:true}); +} diff --git a/tests/portal/test_keycloak_client.py b/tests/portal/test_keycloak_client.py index d214bf0e6..3ba8f7f0f 100644 --- a/tests/portal/test_keycloak_client.py +++ b/tests/portal/test_keycloak_client.py @@ -12,6 +12,10 @@ SCRIPT = Path(__file__).resolve().parents[2] / "charts" / "openshell-rhdh" / "files" / "keycloak-client.py" +import sys +sys.path.insert(0, str(SCRIPT.parent)) + + class FakeKeycloak: def __init__(self, clients=None, users=(), groups=()): self.clients = list(clients or []) @@ -226,7 +230,7 @@ def test_administrators_are_kept_out_of_the_users_group(client, monkeypatch): def test_admin_scope_for_catalog_reader(client, monkeypatch, existing_client, scope_state): monkeypatch.setenv("USERS_GROUP", "saw-users") kc = FakeKeycloak([{"id": "rhdh-id", "clientId": "rhdh"}] if existing_client else []) - scope = {"id": "admin-scope", "name": "kc-admin-api"} + scope = {"id": "admin-scope", "name": "kc-admin-api", "protocol": "openid-connect"} other = {"id": "profile-scope", "name": "profile"} kc.scopes = [other] if scope_state == "absent" else [other, scope] kc.default_scopes = [other] + ([scope] if scope_state == "assigned" else []) @@ -244,7 +248,7 @@ def test_admin_scope_for_catalog_reader(client, monkeypatch, existing_client, sc def test_login_only_client_does_not_get_admin_scope(client, monkeypatch): monkeypatch.delenv("USERS_GROUP", raising=False) kc = FakeKeycloak() - kc.scopes = [{"id": "admin-scope", "name": "kc-admin-api"}] + kc.scopes = [{"id": "admin-scope", "name": "kc-admin-api", "protocol": "openid-connect"}] assert client(kc) == 0 assert kc.default_scopes == [] and kc.scope_puts == [] assert kc.role_mappings == [] diff --git a/tests/portal/test_oidc_startup.py b/tests/portal/test_oidc_startup.py new file mode 100644 index 000000000..7baad65af --- /dev/null +++ b/tests/portal/test_oidc_startup.py @@ -0,0 +1,77 @@ +"""Dependency failures must delay stock RHDH, not poison its OIDC provider.""" +import importlib.util +import io +import json +from pathlib import Path +import ssl +import urllib.error + +import pytest + +SOURCE = Path(__file__).resolve().parents[2] / "charts/openshell-rhdh/files/wait-oidc.py" +spec = importlib.util.spec_from_file_location("wait_oidc", SOURCE) +gate = importlib.util.module_from_spec(spec) +spec.loader.exec_module(gate) +ISSUER = "https://identity.example/realms/saw" + + +def metadata(): + return {"issuer": ISSUER, **{k: ISSUER + "/" + k for k in ( + "authorization_endpoint", "token_endpoint", "userinfo_endpoint", "jwks_uri")}} + + +class Response(io.BytesIO): + status = 200 + + def geturl(self): + return ISSUER + "/.well-known/openid-configuration" + + +def test_verified_discovery_without_credentials(monkeypatch): + context = ssl.create_default_context() + + def request(req, **kwargs): + assert req.full_url == ISSUER + "/.well-known/openid-configuration" + assert req.get_header("Authorization") is None + assert kwargs == {"context": context, "timeout": 10} + assert context.verify_mode == ssl.CERT_REQUIRED and context.check_hostname + return Response(json.dumps(metadata()).encode()) + + monkeypatch.setattr(gate.urllib.request, "urlopen", request) + gate.discover(ISSUER, context) + + +@pytest.mark.parametrize("fault", ["issuer", "endpoint", "non-object", "oversize"]) +def test_bad_metadata_never_unblocks_backend(monkeypatch, fault): + value = metadata() + if fault == "issuer": + value["issuer"] = "https://other.example" + if fault == "endpoint": + value["token_endpoint"] = "http://identity.example/token" + if fault == "non-object": + value = [] + body = b"x" * (1024 * 1024 + 1) if fault == "oversize" else json.dumps(value).encode() + monkeypatch.setattr(gate.urllib.request, "urlopen", lambda *a, **k: Response(body)) + with pytest.raises(ValueError): + gate.discover(ISSUER, ssl.create_default_context()) + + +@pytest.mark.parametrize("recover", [False, True]) +def test_discovery_retries_and_deadline_fail_closed(monkeypatch, recover, capsys): + clock = [0] + attempts = [] + monkeypatch.setattr(gate.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(gate.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds)) + context = ssl.create_default_context() + monkeypatch.setattr(gate.ssl, "create_default_context", lambda **kw: context) + + def discover(issuer, ctx): + assert issuer == ISSUER and ctx is context + attempts.append(clock[0]) + if len(attempts) == 1 or not recover: + raise urllib.error.HTTPError(ISSUER, 503, "private-response", {}, None) + + monkeypatch.setattr(gate, "discover", discover) + assert gate.wait(ISSUER, "/public-ca.crt", timeout=10) == (0 if recover else 1) + assert len(attempts) == (2 if recover else 3) + assert "private-response" not in capsys.readouterr().out diff --git a/tests/portal/test_portal.py b/tests/portal/test_portal.py index cc25242dc..3dc99b405 100644 --- a/tests/portal/test_portal.py +++ b/tests/portal/test_portal.py @@ -142,6 +142,55 @@ def test_the_scaffolders_plugin_token_names_the_user_it_acts_for(portal): assert portal.verify_backstage_token(token, auth.jwks(), scaffolder.jwks()) == "alice" +@pytest.mark.parametrize("url", ["http://rhdh.test/keys", "https://user@rhdh.test/keys"]) +def test_signing_keys_require_verified_https(portal, url, monkeypatch): + monkeypatch.setenv("RHDH_SKIP_VERIFY", "true") + with pytest.raises(portal.PortalError, match="HTTPS endpoint"): + portal.fetch_json(url) + assert portal.NoRedirect().redirect_request(None, None, 302, "", {}, "https://other.test") is None + + +def test_cached_verifier_refreshes_both_rotated_backstage_keys(portal, monkeypatch): + auth_path = "/api/auth/.well-known/jwks.json" + scaffolder_path = "/api/scaffolder/.backstage/auth/v1/jwks.json" + old_auth, old_plugin = Signer("old-auth"), Signer("old-scaffolder") + new_auth, new_plugin = Signer("new-auth"), Signer("new-scaffolder") + issued = {auth_path: old_auth, scaffolder_path: old_plugin} + calls = [] + monkeypatch.setenv("RHDH_INTERNAL_URL", "https://rhdh.test") + + def fetch(url, insecure=False): + assert not insecure + path = url.removeprefix("https://rhdh.test") + calls.append(path) + return issued[path].jwks() + + monkeypatch.setattr(portal, "fetch_json", fetch) + cache = portal.JwksCache() + assert portal.token_user(plugin_token(old_plugin, old_auth.limited()), cache) == "alice" + issued.update({auth_path: new_auth, scaffolder_path: new_plugin}) + signed = plugin_token(new_plugin, new_auth.limited()) + assert portal.token_user(signed, cache) == "alice" + assert portal.token_user(signed, cache) == "alice" + assert calls.count(auth_path) == calls.count(scaffolder_path) == 2 + with pytest.raises(portal.PortalError, match="signature is not valid"): + portal.token_user(plugin_token(new_plugin, Signer("new-auth").limited()), cache) + + +def test_unknown_key_ids_do_not_trigger_unbounded_issuer_fetches(portal, monkeypatch): + real = Signer("published") + calls = [] + monkeypatch.setenv("RHDH_INTERNAL_URL", "https://rhdh.test") + monkeypatch.setattr(portal, "fetch_json", lambda url, insecure=False: calls.append(url) or real.jwks()) + cache = portal.JwksCache() + assert portal.token_user(real.token(), cache) == "alice" + for kid in ("unknown-a", "unknown-b", "unknown-c"): + with pytest.raises(portal.PortalError, match="signing key is not in the JWKS"): + portal.token_user(Signer(kid).token(), cache) + assert len(calls) == 2 + assert portal.token_user(real.token(), cache) == "alice" + + @pytest.mark.parametrize("make, message", [ (lambda auth, sc: plugin_token(Signer("x"), auth.limited()), "scaffolder token: its signing key"), (lambda auth, sc: plugin_token(sc, Signer("auth").limited()), "user token: the signature"), @@ -232,7 +281,21 @@ def test_generator_params_are_one_user_saw_users_values(portal): ws = {"name": "alice", "profiles": ["data-science"], "vaultPrefix": "secret/data/hub/saw-alice"} [params] = portal.generator_params([ws], defaults) assert params["name"] == "alice" - assert json.loads(params["values"]) == {**defaults, "users": [ws]} + assert json.loads(params["values"]) == {**defaults, "users": [ + {**ws, "values": {"deployment": {"mode": "vm"}}}]} + + +def test_operator_pin_selects_application_and_child_source_without_changing_other_owners(portal): + defaults = {"global": {"repoURL": "https://example.com/repo.git", "targetRevision": "legacy"}, + "workspaceRevisions": {"alice": "a"*40}} + users = [{"name": name, "profiles": ["data-science"], "workspaceRevisions": {name: "b"*40}} + for name in ("alice", "bob")] + params = portal.generator_params(users, defaults) + assert [p["revision"] for p in params] == ["a"*40, "legacy"] + for p in params: + values = json.loads(p["values"]) + assert values["global"] == {"repoURL": defaults["global"]["repoURL"], "targetRevision": p["revision"]} + assert "workspaceRevisions" not in values def entity_docs(text): @@ -304,6 +367,7 @@ def __init__(self): self.rv = 0 # last resourceVersion handed out self.deleted = [] self.vault = {} # kv path -> data + self.vault_health = {"initialized": True, "sealed": False, "standby": False} self.logins = [] self.logs = {} # pod log path -> text @@ -324,6 +388,8 @@ def _body(self): def do_GET(self): url = urlsplit(self.path) + if url.path == "/v1/sys/health": + return self._reply(200, fake.vault_health) if url.path in fake.logs: raw = fake.logs[url.path].encode() self.send_response(200) @@ -402,6 +468,43 @@ def request(self, name, data, age=0, labels=None): @pytest.fixture def world(portal, monkeypatch, tmp_path): fake = Fake() + fake.objects["/apis/config.openshift.io/v1/clusterversions/version"] = { + "status": {"conditions": [{"type": "Available", "status": "True"}], + "desired": {"version": "4.22.0"}}} + for crd_name in ("applications.argoproj.io", "applicationsets.argoproj.io", + "sandboxes.agents.x-k8s.io", "externalsecrets.external-secrets.io", + "adminnetworkpolicies.policy.networking.k8s.io", + "kataconfigs.kataconfiguration.openshift.io"): + fake.objects["/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + crd_name] = { + "status": {"conditions": [{"type": "Established", "status": "True"}]}} + fake.objects["/apis/admissionregistration.k8s.io/v1"] = { + "groupVersion": "admissionregistration.k8s.io/v1", + "resources": [{"name": "validatingadmissionpolicies"}, + {"name": "validatingadmissionpolicybindings"}], + } + fake.objects["/apis/storage.k8s.io/v1/storageclasses"] = {"items": [{ + "metadata": {"name": "test-default", + "annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}]} + fake.objects["/apis/external-secrets.io/v1/clustersecretstores"] = {"items": [{ + "metadata": {"name": "vault-backend"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}}]} + fake.objects["/apis/kataconfiguration.openshift.io/v1/kataconfigs"] = {"items": [{ + "status": {"conditions": [{"type": "InProgress", "status": "False"}], + "kataNodes": {"nodeCount": 1, "readyNodeCount": 1, + "installed": ["worker-a"]}}}]} + fake.objects["/api/v1/nodes"] = {"items": [{ + "metadata": {"name": "worker-a", "labels": { + "kubernetes.io/arch": "amd64", "node-role.kubernetes.io/worker": "", + "saw.example.com/kata-pool": "true"}}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, "spec": {}}]} + kata_site = synthetic_preflight_site(portal, {}, "kata") + kata_name = kata_site["kataRuntimeClass"]["name"] + fake.objects["/apis/node.k8s.io/v1/runtimeclasses/" + kata_name] = { + "metadata": {"name": kata_name, "uid": kata_site["kataRuntimeClass"]["uid"], + "labels": {"openshell.ai/sandbox-attachable": "true"}}, + "handler": "kata", "scheduling": {"nodeSelector": {"saw.example.com/kata-pool": "true"}}, + "overhead": {"podFixed": {"memory": "128Mi"}}, + } server, url = fake.serve() auth, scaffolder = Signer("auth"), Signer("scaffolder") sa = tmp_path / "sa" @@ -413,11 +516,23 @@ def world(portal, monkeypatch, tmp_path): scaffolder.jwks() if "/api/scaffolder/" in u else auth.jwks()) generations = iter(GEN) monkeypatch.setattr(portal, "new_generation", lambda: next(generations)) - for k, v in {"NAMESPACE": "saw-portal", "CATALOG_PATH": str(CATALOG), "RHDH_INTERNAL_URL": "http://rhdh", + fixture_catalog = json.loads(CATALOG.read_text()) + fixture_catalog["profiles"]["audited-openclaw"] = json.loads((ROOT/"tests/fixtures/audited-profile.json").read_text()) + catalog_path = tmp_path / "catalog.json" + catalog_path.write_text(json.dumps(fixture_catalog)) + for k, v in {"NAMESPACE": "saw-portal", "CATALOG_PATH": str(catalog_path), "RHDH_INTERNAL_URL": "http://rhdh", "ARGO_NAMESPACE": "vp-gitops", "VAULT_ADDR": url, "VAULT_AUTH_MOUNT": "hub", "VAULT_ROLE": "saw-portal-writer", "VAULT_KV_MOUNT": "secret", "VAULT_PREFIX_BASE": "hub"}.items(): monkeypatch.setenv(k, v) + monkeypatch.setenv("PIPELINE_RUN", "fixture-registration") + (tmp_path / "directory-secret").write_text("synthetic-directory-client") + monkeypatch.setenv("KEYCLOAK_URL", "https://directory.test") + monkeypatch.setenv("KEYCLOAK_REALM", "test") + monkeypatch.setenv("KEYCLOAK_CLIENT_ID", "directory-test") + monkeypatch.setenv("KEYCLOAK_SECRET_FILE", str(tmp_path / "directory-secret")) + monkeypatch.setattr(portal, "keycloak_user_ids", lambda *args, **kwargs: { + "alice": "478fdd1f-8db9-41e8-97ad-b563215c89e9"}) yield fake, Tokens(auth, scaffolder) server.shutdown() @@ -435,6 +550,63 @@ def ds_request(signer, **extra): "inference.api_key": "nvapi-1", "web-search.api_key": "brave-1", **extra} +def synthetic_preflight_site(portal, site, mode="container"): + """A cluster-agnostic passing site for the read-only workspace gate.""" + site = json.loads(json.dumps(site)) + pin = "a" * 40 + image = "registry.example.invalid/saw@sha256:" + "b" * 64 + overrides = site.setdefault("profileImageOverrides", {}) + for profile in ("data-science", "custom-inference", "ci-nvidia"): + overrides.setdefault(profile, {}).setdefault("notebook", image) + site.setdefault("providerAllowHosts", ["api.example.invalid"]) + site.setdefault("providerPolicy", {"allowHosts": ["api.example.invalid"]}) + site.setdefault("stack", { + "identity": {"revision": pin, "values": {"secretStore": "vault-backend"}}, + "policy": {"revision": pin, "values": {"enabled": True}}, + "proxy": {"revision": pin, "values": { + "image": image, "network": {"bootstrapEgress": [{"to": ["api.example.invalid"]}]}, + }}, + "gateway": {"revision": pin, "values": {"image": image}}, + "audit": {"revision": pin, "values": {"image": image}}, + }) + enrollment = site.get("automaticEnrollment") + if isinstance(enrollment, dict): + # These registration tests use foundation mode unless they provide an + # explicit reviewed boundary reference. + enrollment.setdefault("enabled", False) + enrollment.setdefault("revision", pin) + if mode == "kata": + runtime = { + "metadata": {"name": "saw-kata", "uid": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "labels": {"openshell.ai/sandbox-attachable": "true"}}, + "handler": "kata", + "scheduling": {"nodeSelector": {"saw.example.com/kata-pool": "true"}}, + "overhead": {"podFixed": {"memory": "128Mi"}}, + } + site["kataRuntimeClass"] = { + "name": runtime["metadata"]["name"], "uid": runtime["metadata"]["uid"], + "sha256": portal._preflight.runtime_placement_digest(runtime), + } + return site + + +@pytest.mark.parametrize("mode", ["container", "kata"]) +def test_portal_rejects_nontraversable_image_before_vault(portal, world, monkeypatch, capsys, mode): + fake, signer = world + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment": {"approvedProfiles": {"data-science": "a" * 64}}, + }, mode) + del site["profileImageOverrides"]["data-science"]["notebook"] + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", json.dumps(["vm", mode])) + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.request("saw-req-inaccessible", ds_request( + signer, deploymentMode=mode, profile="data-science")) + assert portal.main(["create", "saw-req-inaccessible"]) != 0 + assert "WorkloadImageNotNamespaceAccessible" in capsys.readouterr().out + assert not fake.vault and REGISTRY not in fake.objects + + def test_create_writes_vault_and_the_registry(portal, world): fake, signer = world fake.request("saw-req-1", ds_request(signer)) @@ -446,7 +618,8 @@ def test_create_writes_vault_and_the_registry(portal, world): assert cm["metadata"]["labels"]["saw.redhat.com/workspace"] == "true" assert json.loads(cm["data"]["user.json"]) == { "name": "alice", "profiles": ["data-science"], "ownerSubject": "", - "vaultPrefix": f"secret/data/{KEYS}", "pruneOnRemove": True} + "vaultPrefix": f"secret/data/{KEYS}", "pruneOnRemove": True, + "values": {"deployment": {"mode": "vm"}}} assert "/api/v1/namespaces/saw-portal/secrets/saw-req-1" not in fake.objects, "request consumed" @@ -967,6 +1140,74 @@ def ping(url): "https://alice-default-notebook-ui.apps.example.com"] +@pytest.mark.parametrize("reason,rpc_code,expected", [ + ("GatewayMismatch", None, "Container reconciliation failed (GatewayMismatch)"), + ("CoreRpcRejected", "FAILED_PRECONDITION", + "Container reconciliation failed (OpenShell FAILED_PRECONDITION)"), + ("private-token", None, "Container reconciliation failed"), + (["private-token"], None, "Container reconciliation failed"), +]) +def test_container_failure_report_shows_only_safe_reason_codes(portal, reason, rpc_code, expected): + workspace = portal._workspace + + def digest(value): + encoded = workspace.json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + return workspace.hashlib.sha256(encoded).hexdigest() + + files = {"profiles__data-science__default__workspace.yaml": "workspace source"} + profile_uid = "profile-uid" + intent = { + "schemaVersion": 1, + "generation": 1, + "desiredState": "Running", + "profileRef": {"name": "saw-bom-profiles", "uid": profile_uid, + "sha256": digest(files)}, + "bindings": {"ownerSubject": "owner-subject", "namespace": "saw-alice", + "mode": "container", "ownerID": "installation-id"}, + } + runtime_status = { + "schemaVersion": 1, + "observedGeneration": 1, + "inputRevision": digest({"intent": intent, "files": files}), + "phase": "Blocked", + "reason": reason, + } + if rpc_code: + runtime_status["rpcCode"] = rpc_code + app = { + "metadata": {"labels": {"openshell.pattern/owner": "alice"}}, + "status": {"sync": {"status": "Synced"}, "health": {"status": "Healthy"}, + "operationState": {"phase": "Succeeded"}}, + } + resources = { + "/apis/argoproj.io/v1alpha1/namespaces/vp-gitops/applications/saw-alice": app, + "/api/v1/namespaces/saw-alice/configmaps/saw-runtime-intent": { + "metadata": {"labels": {"saw.redhat.com/runtime-owner": "installation-id"}}, + "data": {"intent.json": json.dumps(intent)}, + }, + "/api/v1/namespaces/saw-alice/configmaps/saw-runtime-status": { + "metadata": {"labels": {"saw.redhat.com/runtime-owner": "installation-id", + "saw.redhat.com/managed-by": "kubernetes-runtime-controller"}}, + "data": {"status.json": json.dumps(runtime_status)}, + }, + "/api/v1/namespaces/saw-alice/configmaps/saw-bom-profiles": { + "metadata": {"uid": profile_uid}, "data": files, + }, + } + entry = {"name": "alice", "registration": "generation-1", "installationID": "installation-id", + "intentGeneration": 1, + "profiles": ["data-science"], "desiredState": "Running", + "values": {"deployment": {"mode": "container"}}} + + result = workspace.container_status( + lambda path: resources.get(path), entry, {"subject": "owner-subject"}, + "vp-gitops", "example.com", lambda _url: False, + ) + assert result["failed"] is True + assert result["message"] == expected + assert "private-token" not in result["message"] + + def test_sandbox_uis_do_not_hide_an_unavailable_openshell_webui(portal, world): fake = world[0] fake.objects[REGISTRY] = {"metadata": {"name": "saw-ws-alice"}, @@ -1435,6 +1676,68 @@ def test_owner_subject_cache_rechecks_account_recreation(portal, monkeypatch, tm assert subjects.fill(workspaces)[0]["ownerSubject"] == "" +@pytest.mark.parametrize("failure", [401, "network"]) +def test_owner_lookup_failure_holds_vm_and_container_projection_until_recovery( + portal, monkeypatch, tmp_path, failure): + secret = tmp_path / "secret" + secret.write_text("test-client-secret\n") + for key, value in {"KEYCLOAK_URL": "https://kc.example.com", "KEYCLOAK_REALM": "openshell", + "KEYCLOAK_CLIENT_ID": "rhdh", "KEYCLOAK_SECRET_FILE": str(secret)}.items(): + monkeypatch.setenv(key, value) + now = [1000] + monkeypatch.setattr(portal.time, "time", lambda: now[0]) + failed, calls = [False], [] + + def lookup(*args, **kwargs): + calls.append(list(args[4])) + if failed[0]: + if failure == "network": + raise OSError("connection refused") + raise portal.HttpError(failure, "Keycloak users: HTTP 401") + return {name: "id-" + name for name in args[4]} + + monkeypatch.setattr(portal, "keycloak_user_ids", lookup) + subjects = portal.OwnerSubjects(ttl=60) + workspaces = [{"name": "vm-owner", "ownerSubject": "", "values": {"deployment": {"mode": "vm"}}}, + {"name": "container-owner", "ownerSubject": "", "values": {"deployment": {"mode": "container"}}}] + assert [w["ownerSubject"] for w in subjects.fill(workspaces)] == ["id-vm-owner", "id-container-owner"] + failed[0] = True + now[0] += 60 + with pytest.raises(portal.PortalError, match="workspace projection held"): + subjects.fill(workspaces) + # No stale identities, empty owner fields or partial generator response + # may escape while the failed request is being throttled. + now[0] += 1 + for selected in (workspaces, workspaces[:1], workspaces[1:]): + with pytest.raises(portal.PortalError, match="workspace projection held"): + subjects.fill(selected) + assert len(calls) == 2 + failed[0] = False + now[0] += 60 + assert [w["ownerSubject"] for w in subjects.fill(workspaces)] == ["id-vm-owner", "id-container-owner"] + assert len(calls) == 3 + + +def test_initial_owner_lookup_failure_is_not_cached_as_missing_user(portal, monkeypatch, tmp_path): + secret = tmp_path / "secret" + secret.write_text("test-client-secret\n") + for key, value in {"KEYCLOAK_URL": "https://kc.example.com", "KEYCLOAK_REALM": "openshell", + "KEYCLOAK_CLIENT_ID": "rhdh", "KEYCLOAK_SECRET_FILE": str(secret)}.items(): + monkeypatch.setenv(key, value) + def unavailable(*args, **kwargs): + raise portal.HttpError(401, "Keycloak users: HTTP 401") + + monkeypatch.setattr(portal, "keycloak_user_ids", unavailable) + subjects = portal.OwnerSubjects(ttl=60) + workspaces = [{"name": "alice", "ownerSubject": ""}] + for _ in range(2): + with pytest.raises(portal.PortalError, match="workspace projection held"): + subjects.fill(workspaces) + # A separately pinned explicit owner does not depend on failed discovery. + explicit = [{"name": "bob", "ownerSubject": "operator-resolved-subject"}] + assert subjects.fill(explicit) == explicit + + def test_the_ca_bundle_holds_every_source_found(tmp_path, monkeypatch): """files/ca-bundle.py: system CAs, then the cluster's, the extra ones; a missing source is skipped (no service account here).""" @@ -1456,3 +1759,687 @@ def test_the_ca_bundle_holds_every_source_found(tmp_path, monkeypatch): text = out.read_text() assert text.index("SYSTEM") < text.index("TRUSTED") < text.index("EXTRA") assert text.count("BEGIN CERTIFICATE") == 3 + + +# -- deployment selection ------------------------------------------------------ + +def test_deployment_request_defaults_to_vm(portal, monkeypatch): + monkeypatch.delenv("SAW_DEPLOYMENT_MODES", raising=False) + assert portal.request_deployment_mode({}) == "vm" + + +@pytest.mark.parametrize("mode", ["container", "kata"]) +def test_mode_requires_administrator_allowlist(portal, monkeypatch, mode): + monkeypatch.delenv("SAW_DEPLOYMENT_MODES", raising=False) + with pytest.raises(portal.PortalError, match="not enabled"): + portal.request_deployment_mode({"deploymentMode": mode}) + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", json.dumps(['vm', mode])) + assert portal.request_deployment_mode({"deploymentMode": mode}) == mode + + +@pytest.mark.parametrize("mode", ["confidential", "typo", "", None, False, [], {}]) +def test_unknown_or_malformed_mode_is_rejected(portal, mode): + with pytest.raises(portal.PortalError): + portal.request_deployment_mode({"deploymentMode": mode}) + + +@pytest.mark.parametrize("raw", ["vm", "{}", "[]", '["container"]', '["vm","vm"]', + '["vm",{}]', '["vm","confidential"]']) +def test_invalid_administrator_mode_policy_fails_closed(portal, monkeypatch, raw): + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", raw) + with pytest.raises(portal.PortalError, match="SAW_DEPLOYMENT_MODES"): + portal.request_deployment_mode({}) + + +@pytest.mark.parametrize("mode", ["container"]) +def test_unqualified_mode_refused_before_vault_even_when_allowlisted(portal, world, monkeypatch, mode): + fake, signer = world + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm", "container"]') + fake.request("saw-req-mode", ds_request(signer, deploymentMode=mode)) + assert portal.main(["create", "saw-req-mode"]) == 1 + assert not fake.vault and not fake.logins + assert REGISTRY not in fake.objects + + +def test_mode_change_is_refused_before_any_credential_update(portal, world, monkeypatch): + fake, signer = world + fake.request("saw-req-1", ds_request(signer)) + assert portal.main(["create", "saw-req-1"]) == 0 + previous = json.dumps(fake.vault, sort_keys=True) + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm", "container"]') + fake.request("saw-req-2", ds_request(signer, deploymentMode="container", + **{"inference.api_key": "replacement"})) + assert portal.main(["create", "saw-req-2"]) == 1 + assert json.dumps(fake.vault, sort_keys=True) == previous + assert len(fake.logins) == 1 + assert portal.entry_deployment_mode(portal.read_entry(fake.objects[REGISTRY])) == "vm" + + +def test_concurrent_create_cannot_replace_a_different_mode(portal, world): + fake, _ = world + winner = portal.registry_entry("alice", "data-science", GEN[0], "container") + portal.put_configmap(portal.kube(), "saw-portal", "saw-ws-alice", + {"user.json": json.dumps(winner)}, {portal.WORKSPACE_LABEL: "true"}) + stale = portal.registry_entry("alice", "data-science", GEN[1], "vm") + with pytest.raises(portal.PortalError, match="immutable"): + portal.put_configmap(portal.kube(), "saw-portal", "saw-ws-alice", + {"user.json": json.dumps(stale)}, deployment_mode="vm") + assert portal.read_entry(fake.objects[REGISTRY]) == winner + + +@pytest.mark.parametrize("mode", ["vm", "container"]) +def test_mode_serializes_through_registry_to_gitops(portal, mode): + entry = portal.registry_entry("alice", "data-science", GEN[0], mode) + if mode == "container": + with pytest.raises(portal.PortalError, match="container site is missing"): + portal.generator_params([entry], {}) + return + [params] = portal.generator_params([entry], {"defaults": {"openshellSaw": { + "deployment": {"mode": "container"}}}}) + values = json.loads(params["values"]) + assert values["users"][0]["values"]["deployment"]["mode"] == mode + + +def test_legacy_registry_mode_does_not_inherit_changed_chart_default(portal): + entry = {"name": "alice", "profiles": ["data-science"]} + [params] = portal.generator_params([entry], {"defaults": {"openshellSaw": { + "deployment": {"mode": "container"}}}}) + assert json.loads(params["values"])["users"][0]["values"]["deployment"]["mode"] == "vm" + assert "values" not in entry, "normalization must not mutate the input" + + +@pytest.mark.parametrize("mode", ["container"]) +def test_unqualified_runtime_never_uses_vm_readiness(portal, mode): + entry = portal.registry_entry("alice", "data-science", GEN[0], mode) + class RegistryOnly: + def call(self, method, path, **kwargs): + assert method == "GET" and path == REGISTRY + return {"metadata": {"name": "saw-ws-alice"}, + "data": {"user.json": json.dumps(entry)}} + status = portal.workspace_status(RegistryOnly(), "alice", {}, "saw-portal", "vp-gitops", "example.test") + assert status["phase"] == "runtime-not-qualified" + assert status["ready"] is False and status["failed"] is True + assert not status["links"] + + +def test_linked_container_catalog_entry_is_owned_and_links_only_to_the_site(portal, catalog, monkeypatch): + monkeypatch.setenv("SAW_LINKED_WORKSPACES", json.dumps([{"owner":"alice","routeLabel":"alice-agent-ui","title":"OpenClaw"}])) + docs = entity_docs(portal.entities_yaml([], catalog, "example.com", "")) + entry = next(d for d in docs if d["metadata"]["name"] == "linked-alice-agent-ui") + assert entry["spec"]["owner"] == "user:default/alice" + assert entry["metadata"]["links"] == [{"title":"OpenClaw UI","icon":"web","url":"https://alice-agent-ui.apps.example.com"}] + + +def test_linked_runtime_tracks_owner_and_current_generation(portal): + ref = {"namespace":"runtime", "intent":"intent", "status":"status", "installationID":"instance", + "ownerSubject":"subject", "mode":"container"} + status = {"phase":"GenericReady", "observedGeneration":3} + intent = {"generation":3, "desiredState":"Running", "bindings":{ + "namespace":"runtime", "ownerID":"instance", "ownerSubject":"subject", "mode":"container"}} + class Reader: + def call(self, method, path, ok404): + assert method == "GET" and ok404 + assert path in ["/api/v1/namespaces/runtime/configmaps/" + n for n in ("intent","status")] + n = path.rsplit("/", 1)[1] + labels = {"saw.redhat.com/runtime-owner":"instance"} + if n == "status": + labels["saw.redhat.com/managed-by"] = "kubernetes-runtime-controller" + return {"metadata":{"labels":labels}, "data":{n+".json":json.dumps(intent if n=="intent" else status)}} + reader = Reader() + assert portal.linked_runtime_phase(reader, {"runtime":ref}) == "Running" + intent["generation"] += 1 + assert portal.linked_runtime_phase(reader, {"runtime":ref}) == "Reconciling" + intent["bindings"]["ownerSubject"] = "someone-else" + assert portal.linked_runtime_phase(reader, {"runtime":ref}) == "Unknown" + + +@pytest.mark.parametrize("fault,expected", [ + ("profile", "approved profiles: audited-openclaw"), + ("cleanup", "enable portal.pruneOnRemove"), +]) +def test_container_preflight_explains_profile_and_cleanup_failures(portal, world, monkeypatch, capsys, fault, expected): + fake, signer = world + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","container"]') + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a"*64}}, + }) + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + monkeypatch.setenv("PRUNE_ON_REMOVE", "false" if fault == "cleanup" else "true") + fake.request("saw-req-rejected", ds_request(signer, deploymentMode="container", + profile="data-science" if fault == "profile" else "audited-openclaw")) + assert portal.main(["create", "saw-req-rejected"]) != 0 + captured = capsys.readouterr() + assert expected in captured.out + captured.err + assert not fake.vault and REGISTRY not in fake.objects + + +@pytest.mark.parametrize("mode", ["container", "kata"]) +def test_portal_accepts_documented_site_without_generated_provider_policy( + portal, world, monkeypatch, mode): + fake, signer = world + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a" * 64}}, + }, mode) + del site["providerPolicy"] + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", json.dumps(["vm", mode])) + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.request("saw-req-p1234", ds_request( + signer, deploymentMode=mode, profile="audited-openclaw")) + assert portal.main(["create", "saw-req-p1234"]) == 0 + assert REGISTRY in fake.objects + + +@pytest.mark.parametrize("mode", ["container", "kata"]) +def test_container_portal_register_update_delete(portal, world, monkeypatch, mode): + fake, signer = world + subject = "478fdd1f-8db9-41e8-97ad-b563215c89e9" + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", json.dumps(["vm", mode])) + site = synthetic_preflight_site(portal, { + "subject": subject, "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a"*64}}, + }, mode) + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + for i in [1, 2]: + fake.request("saw-req-" + str(i), ds_request(signer, deploymentMode=mode, + profile="audited-openclaw", **{"inference.api_key":f"test-{i}"})) + assert portal.main(["create", "saw-req-" + str(i)]) == 0 + entry = portal.read_entry(fake.objects[REGISTRY]) + assert entry["intentGeneration"] == i and portal.entry_generation("alice", entry) == GEN[0] + assert fake.vault == {f"{KEYS}/inference":{"api_key":f"test-{i}", "provider":"openai"}} + assert portal.generator_params([{**entry,"ownerSubject":"other-subject"}], {}) == [] + output = portal.generator_params([{**entry,"ownerSubject":subject}], {}, + catalog=portal.load_catalog(os.environ["CATALOG_PATH"]))[0] + user = json.loads(output["values"])["users"][0] + assert user["values"]["deployment"]["mode"] == mode + assert user["installationID"] == portal.installation_id("alice", GEN[0]) + assert "test-" not in output["values"] + fake.request("saw-req-3", {"token":signer.token(), "action":"delete", "workspace":"component:default/saw-alice"}) + assert portal.main(["delete","saw-req-3"]) == 0 + ns = f"/api/v1/namespaces/saw-ctl-{GEN[0]}" + fake.objects[ns] = {"metadata":{"name":ns.split("/")[-1]}} + gone, detail = portal.removal_status(portal.kube(), "alice", "vp-gitops", entry) + assert not gone and ns.split("/")[-1] in detail + del fake.objects[ns] + assert portal.removal_status(portal.kube(), "alice", "vp-gitops", entry)[0] + assert portal.main(["finish-delete","alice"]) == 0 + assert not fake.vault and REGISTRY not in fake.objects + + +def test_kata_portal_preflight_rejects_changed_runtime_before_vault(portal, world, monkeypatch, capsys): + fake, signer = world + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a" * 64}}, + }, "kata") + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","kata"]') + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + runtime_name = site["kataRuntimeClass"]["name"] + fake.objects[f"/apis/node.k8s.io/v1/runtimeclasses/{runtime_name}"]["handler"] = "runc" + fake.request("saw-req-katabad", ds_request( + signer, deploymentMode="kata", profile="audited-openclaw")) + + assert portal.main(["create", "saw-req-katabad"]) == 1 + assert "Kata preflight failed: RuntimeClassHandlerMismatch" in capsys.readouterr().out + assert not fake.vault and REGISTRY not in fake.objects + + +def test_first_kata_registration_checks_approval_shape_without_existing_namespace(portal): + site = synthetic_preflight_site(portal, { + "automaticEnrollment": { + "enabled": True, + "reference": { + "validation": "a" * 64, + "binding": "b" * 64, + "sandboxPlacement": "c" * 64, + "sandboxPlacementBinding": "d" * 64, + }, + }, + }, "kata") + + class EmptyKubernetes: + def __init__(self): + self.paths = [] + + def call(self, method, path, **kwargs): + self.paths.append(path) + assert method == "GET" + assert kwargs.get("ok404") is True + return None + + reader = EmptyKubernetes() + portal.check_live_boundary_approval( + reader, "alice", site, "kata", "audited-openclaw", {}) + assert reader.paths == ["/api/v1/namespaces/saw-alice"] + + +def test_first_kata_registration_rejects_missing_approval_before_namespace_read(portal): + site = synthetic_preflight_site(portal, { + "automaticEnrollment": {"enabled": True, "reference": {"validation": "a" * 64}}, + }, "kata") + + class EmptyKubernetes: + def call(self, *args, **kwargs): + pytest.fail("The missing approval must fail before a cluster read") + + with pytest.raises(portal.PortalError, match="BoundaryApprovalMissing"): + portal.check_live_boundary_approval( + EmptyKubernetes(), "alice", site, "kata", "audited-openclaw", {}) + + +def install_live_boundary_fixtures(portal, fake, site, mode="kata"): + owner = "alice" + registration = GEN[0] + owner_id = portal.installation_id(owner, registration) + fake.objects["/api/v1/namespaces/saw-" + owner] = { + "metadata": {"name": "saw-" + owner, "labels": { + portal.PORTAL_NS_LABEL: "true", + "saw.redhat.com/registration": registration, + "saw.redhat.com/runtime-owner": owner_id, + }}, + } + catalog = portal.load_catalog(os.environ["CATALOG_PATH"]) + slots = sorted(portal.sandbox_targets( + owner, registration, catalog["audited-openclaw"])) + relays = [portal.hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] + if len(slots) > 1 else registration for slot in slots] + policy = "saw-saw-" + owner + "-runtime" + validation_spec = { + "controlNamespace": "saw-ctl-" + registration, + "installation": owner_id, + "mode": mode, + "runtimeClass": site.get("kataRuntimeClass", {}).get("name", ""), + "gatewaySlots": [owner + "-" + relay[:8] for relay in relays] if len(slots) > 1 else [], + "gateRelays": ["gate-" + relay + "-claims" for relay in relays], + "browserProofs": ["ui-" + relay + "-browser-proof" for relay in relays], + } + validation_path = "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies/" + policy + fake.objects[validation_path] = { + "kind": "ValidatingAdmissionPolicy", + "metadata": {"name": policy, "generation": 1}, + "spec": validation_spec, + "status": {"observedGeneration": 1, "typeChecking": {"expressionWarnings": []}}, + } + binding_spec = {"policyName": policy, "validationActions": ["Deny"]} + binding_path = "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings/" + policy + fake.objects[binding_path] = { + "kind": "ValidatingAdmissionPolicyBinding", + "metadata": {"name": policy, "generation": 1}, + "spec": binding_spec, + } + reference = { + "validation": portal._preflight.validation_policy_digest( + validation_spec, owner, registration, owner_id, slots), + "binding": portal._preflight.fingerprint(binding_spec), + } + if mode == "kata": + placement_name = policy + "-kata" + placement_spec = {"runtimeClassName": site["kataRuntimeClass"]["name"]} + placement_binding_spec = {"policyName": placement_name, "validationActions": ["Deny"]} + fake.objects["/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicies/" + placement_name] = { + "kind": "ValidatingAdmissionPolicy", + "metadata": {"name": placement_name, "generation": 1}, + "spec": placement_spec, + "status": {"observedGeneration": 1, "typeChecking": {"expressionWarnings": []}}, + } + fake.objects["/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings/" + placement_name] = { + "kind": "ValidatingAdmissionPolicyBinding", + "metadata": {"name": placement_name, "generation": 1}, + "spec": placement_binding_spec, + } + reference["sandboxPlacement"] = portal._preflight.fingerprint(placement_spec) + reference["sandboxPlacementBinding"] = portal._preflight.fingerprint(placement_binding_spec) + site["automaticEnrollment"]["enabled"] = True + site["automaticEnrollment"]["reference"] = reference + return validation_path + + +@pytest.mark.parametrize("change", ["validation", "binding"]) +def test_kata_portal_preflight_rejects_changed_admission_approval_before_vault( + portal, world, monkeypatch, capsys, change): + fake, signer = world + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a" * 64}}, + }, "kata") + validation_path = install_live_boundary_fixtures(portal, fake, site) + if change == "validation": + fake.objects[validation_path]["spec"]["runtimeClass"] = "unapproved-runtime" + else: + fake.objects["/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings/" + "saw-saw-alice-runtime"]["spec"]["validationActions"] = ["Audit"] + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","kata"]') + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.request("saw-req-boundarybad", ds_request( + signer, deploymentMode="kata", profile="audited-openclaw")) + + assert portal.main(["create", "saw-req-boundarybad"]) == 1 + output = capsys.readouterr().out + assert "Workspace preflight failed: BoundaryApprovalChanged" in output + assert not fake.vault and REGISTRY not in fake.objects + + +def test_kata_portal_preflight_accepts_the_approved_live_boundary( + portal, world, monkeypatch): + fake, signer = world + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a" * 64}}, + }, "kata") + install_live_boundary_fixtures(portal, fake, site) + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","kata"]') + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.request("saw-req-boundaryok", ds_request( + signer, deploymentMode="kata", profile="audited-openclaw")) + + assert portal.main(["create", "saw-req-boundaryok"]) == 0 + assert fake.vault and REGISTRY in fake.objects + + +def test_container_portal_preflight_rejects_sealed_vault_before_vault_write(portal, world, monkeypatch, capsys): + fake, signer = world + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a" * 64}}, + }) + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","container"]') + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.vault_health = {"initialized": True, "sealed": True} + fake.request("saw-req-vaultbad", ds_request( + signer, deploymentMode="container", profile="audited-openclaw")) + + assert portal.main(["create", "saw-req-vaultbad"]) == 1 + assert "Workspace preflight failed: SecretServicesUnavailable" in capsys.readouterr().out + assert not fake.logins and not fake.vault and REGISTRY not in fake.objects + +@pytest.mark.parametrize("mode,ready", [("registered",True),("pending",False),("wrong-client",False),("foreign",False),("anonymous-chat",False)]) +def test_chat_readiness_requires_registered_owner_callback(portal, monkeypatch, mode, ready): + import io + from urllib.error import HTTPError + from urllib.parse import urlencode + origin, issuer = "https://chat.test", "https://id.test" + monkeypatch.setenv("KEYCLOAK_URL",issuer) + monkeypatch.setenv("KEYCLOAK_REALM","openshell") + class Response(io.BytesIO): + status = 200 + class Opener: + def open(self, url, timeout): + if url==origin: + if mode=="anonymous-chat": return Response(b"chat") + params=urlencode(dict(client_id=("wrong" if mode=="wrong-client" else "openshell-dashboard"),redirect_uri=origin+"/oauth2/callback")) + host="https://other.test" if mode=="foreign" else issuer + raise HTTPError(url,302,"",{"Location":host+"/realms/openshell/protocol/openid-connect/auth?"+params},None) + assert url.startswith(issuer) + if mode=="pending": raise HTTPError(url,400,"Invalid redirect URI",{},None) + return Response(b'
') + monkeypatch.setattr(portal.urllib.request,"build_opener",lambda *args:Opener()) + assert portal.owner_route_available(origin) is ready + + +def test_generator_publishes_only_immutable_owner_bound_data(portal): + reg="a"*16; owner="9aaea20f-245f-407a-a0c3-fbd894615b90" + sub="e05c1559-5009-4663-81fb-3c5facb5ced8" + ws=dict(name="ada",registration=reg,installationID=owner,ownerSubject=sub) + a=dict(image="pinned@sha256:abc",revision="fixed",buildNamespace="trust",profileSHA256="bom", + reference={"gateway":{}},relay={"issuer":"https://issuer","revision":"fixed"}) + site={"automaticEnrollment":a} + intent=dict(bindings=dict(ownerID=owner,namespace="saw-ada",ownerSubject=sub),profileRef=dict(sha256="bom")) + body=dict(registration=reg,intent=intent,approved={"gateway":{}},gatewayCA="public") + meta={"labels":{"saw.redhat.com/runtime-owner":owner}} + cm=dict(immutable=True,metadata=meta,data={}) + relay=dict(immutable=True,metadata=meta,data={"subject":"e5ed3695-4476-4b1f-8e5d-daf5f6ccfae3"}) + def read(path): + assert path.startswith("/api/v1/namespaces/saw-ctl-"+reg+"/configmaps/") + cm["data"]={"binding.json":json.dumps(body)} + return relay if path.endswith("relay-subject") else cm + assert portal.enrolled_site(site,ws,read)["enrollment"]["relay"]["subject"]==relay["data"]["subject"] + body["revision"]="caller-controlled" + with pytest.raises(portal.PortalError):portal.enrolled_site(site,ws,read) + del body["revision"] + cm["immutable"]=False + with pytest.raises(portal.PortalError):portal.enrolled_site(site,ws,read) + cm["immutable"]=True;intent["bindings"]["ownerID"]="foreign" + with pytest.raises(portal.PortalError):portal.enrolled_site(site,ws,read) + +def test_model_input_cannot_publish_a_submitted_credential(portal): + profile={"workspaces":[{"name":"research","enabled":True,"sandboxes":[ + {"name":"notebook","enabled":True,"modelInput":"provider.choice"}]}], + "secrets":{"provider":{"fields":[{"key":"choice","kind":"text"},{"key":"api_key","kind":"secret"}]}}} + inputs={"provider":{"api_key":"synthetic-private-value","choice":"model-x"}} + assert portal.public_model_inputs(profile, inputs)=={"provider.choice":"model-x"} + inputs["provider"]["choice"]=inputs["provider"]["api_key"] + with pytest.raises(portal.PortalError,match="invalid public model selector"): + portal.public_model_inputs(profile, inputs) + + +@pytest.mark.parametrize("field", ["model", "cuda_model"]) +def test_public_nvidia_selector_uses_its_catalog_field(portal, field): + catalog = json.loads((PORTAL.parent / "profile-catalog.json").read_text()) + profile = catalog["profiles"]["data-science"] + default = next(f["default"] for f in profile["secrets"]["inference"]["fields"] + if f["key"] == field) + assert "/" in default + inputs = {"inference": {field: default, "api_key": "private-test-key"}} + assert portal.public_model_inputs(profile, inputs) == {"inference." + field: default} + for invalid in ("unapproved/model", "https://malicious.test/model", inputs["inference"]["api_key"]): + inputs["inference"][field] = invalid + with pytest.raises(portal.PortalError, match="invalid public model selector"): + portal.public_model_inputs(profile, inputs) + + +def test_owner_state_change_is_generation_scoped_and_does_not_touch_vault(portal, world, monkeypatch): + fake, signer = world + site = synthetic_preflight_site(portal, { + "subject": "478fdd1f-8db9-41e8-97ad-b563215c89e9", "modelLifecycle": True, + "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a"*64}}, + }) + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","container"]') + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.request("saw-req-1", ds_request(signer, deploymentMode="container", profile="audited-openclaw")) + assert portal.main(["create", "saw-req-1"]) == 0 + entry = portal.read_entry(fake.objects[REGISTRY]) + registration = portal.entry_generation("alice", entry) + vault_before = json.dumps(fake.vault, sort_keys=True) + def state(name, operation, generation, **extra): + fake.request(name, {"action": "create", "token": signer.token(), "runtimeAction": operation, + "registration": registration, "expectedGeneration": "g-" + str(generation), **extra}) + return portal.main(["create", name]) + assert state("saw-req-2", "stop", 1, forUser="bob") != 0 + assert state("saw-req-3", "stop", 1, registration="0"*16) != 0 + assert state("saw-req-4", "stop", 1) == 0 + assert portal.read_entry(fake.objects[REGISTRY])["desiredState"] == "Stopped" + assert state("saw-req-5", "start", 1) != 0 + assert state("saw-req-6", "start", 2) == 0 + assert portal.read_entry(fake.objects[REGISTRY]) == {**entry, "intentGeneration": 3} + assert json.dumps(fake.vault, sort_keys=True) == vault_before + + +def test_custom_endpoint_is_approved_before_writing_vault(portal, world, monkeypatch): + fake, signer = world + url="https://models.fixture.test/v1" + boundary={"allowHosts":["models.fixture.test"],"inferenceBaseURL":url} + site=synthetic_preflight_site(portal, { + "subject":"478fdd1f-8db9-41e8-97ad-b563215c89e9", + "profileBoundaries":{"custom-inference":boundary}, + "automaticEnrollment":{"approvedProfiles":{"custom-inference":"a"*64}}, + }) + monkeypatch.setenv("SAW_DEPLOYMENT_MODES",'["vm","container"]') + monkeypatch.setenv("SAW_CONTAINER_SITES",json.dumps({"alice":site})) + def submit(name,endpoint): + request = ds_request(signer,deploymentMode="container",profile="custom-inference", + **{"inference.url":endpoint,"inference.model":"org/model","inference.api_key":"private-test-key"}) + request.pop("web-search.api_key") + fake.request(name,request) + return portal.main(["create",name]) + assert submit("saw-req-bad","https://elsewhere.example/v1") != 0 + assert not fake.vault and REGISTRY not in fake.objects + assert submit("saw-req-good",url) == 0 + entry=portal.read_entry(fake.objects[REGISTRY]) + assert entry["customInferenceURL"]==url and entry["modelInputs"]=={"inference.model":"org/model"} + assert "private-test-key" not in json.dumps(entry) + assert entry["webSearch"] == {"enabled":False} and not any("web-search" in k for k in fake.vault) + + +def test_headless_container_requires_current_runtime_and_healthy_argo(portal, monkeypatch): + import hashlib + reg="0123456789abcdef" + subject="478fdd1f-8db9-41e8-97ad-b563215c89e9" + monkeypatch.setenv("SAW_CONTAINER_SITES",json.dumps({"alice":{"subject":subject,"automaticEnrollment":{"approvedProfiles":{"headless-test":"a"*64}}}})) + entry=portal.registry_entry("alice","headless-test",reg,"container") + entry["intentGeneration"]=1 + install=portal.installation_id("alice",reg) + digest=lambda value: hashlib.sha256(json.dumps(value,sort_keys=True,separators=(",",":")).encode()).hexdigest() + files={"fixture":"profile"} + intent={"generation":1,"desiredState":"Running","profileRef":{"name":"saw-bom-profiles","uid":"bom","sha256":digest(files)}, + "bindings":{"ownerSubject":subject,"namespace":"saw-alice","mode":"container","ownerID":install}} + state={"phase":"GenericReady","observedGeneration":1,"inputRevision":digest({"intent":intent,"files":files})} + def cm(key,value): + return {"metadata":{"labels":{"saw.redhat.com/runtime-owner":install,"saw.redhat.com/managed-by":"kubernetes-runtime-controller"}}, + "data":{key:json.dumps(value)}} + objects={"saw-alice":{"metadata":{"labels":{"openshell.pattern/owner":"alice"}}, + "status":{"sync":{"status":"Synced"},"health":{"status":"Healthy"}}}, + "saw-runtime-intent":cm("intent.json",intent),"saw-runtime-status":cm("status.json",state), + "saw-bom-profiles":{"metadata":{"uid":"bom"},"data":files}} + class Reader: + def call(self,method,path,ok404): + return objects[path.rsplit("/",1)[1]] + profile = {"headless-test": {"workspaces": [{ + "name": "default", "enabled": True, + "sandboxes": [{"name": "worker", "enabled": True, "uiRoute": False}], + }]}} + def status(): + return portal.container_status(Reader(),"alice",entry,"argo","example.test", + lambda _: pytest.fail("headless registration has no route"),profile) + assert status()["ready"] and not status()["links"] + objects["saw-runtime-status"]=cm("status.json",{**state,"observedGeneration":0}) + assert not status()["ready"] + + +def test_recycled_container_login_cannot_write_or_delete(portal, world, monkeypatch): + fake, signer = world + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","container"]') + site = synthetic_preflight_site(portal, { + "subject":"478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment":{"approvedProfiles":{"audited-openclaw":"a"*64}}, + }) + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.request("saw-req-owner", ds_request(signer, deploymentMode="container", profile="audited-openclaw")) + assert portal.main(["create","saw-req-owner"]) == 0 + before = json.dumps([fake.vault, fake.objects[REGISTRY]], sort_keys=True) + monkeypatch.setattr(portal, "keycloak_user_ids", lambda *a, **kw: {"alice":"different-subject"}) + for action in ["create","delete"]: + data = ds_request(signer, deploymentMode="container", profile="audited-openclaw") + fake.request("saw-req-reused", {**data, "action":action}) + assert portal.main([action,"saw-req-reused"]) == 1 + assert json.dumps([fake.vault, fake.objects[REGISTRY]], sort_keys=True) == before + + +def test_failed_vault_write_retains_reserved_generation(portal, world, monkeypatch): + fake, signer = world + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","container"]') + site = synthetic_preflight_site(portal, { + "subject":"478fdd1f-8db9-41e8-97ad-b563215c89e9", + "automaticEnrollment":{"approvedProfiles":{"audited-openclaw":"a"*64}}, + }) + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + real = portal.vault_client + def unavailable(): + raise portal.PortalError("simulated Vault outage") + monkeypatch.setattr(portal, "vault_client", unavailable) + data = ds_request(signer, deploymentMode="container", profile="audited-openclaw") + fake.request("saw-req-reserve", data) + assert portal.main(["create","saw-req-reserve"]) == 1 + entry = portal.read_entry(fake.objects[REGISTRY]) + assert portal.entry_generation("alice", entry) == GEN[0] + assert not fake.vault + assert portal.list_workspaces(portal.kube(), "saw-portal") == [] + assert portal.list_workspaces(portal.kube(), "saw-portal", deleting=True) == [entry] + monkeypatch.setattr(portal, "vault_client", real) + fake.request("saw-req-resume", data) + assert portal.main(["create","saw-req-resume"]) == 0 + assert portal.entry_generation("alice", portal.read_entry(fake.objects[REGISTRY])) == GEN[0] + assert list(fake.vault) == [f"{KEYS}/inference"] + + +def test_incomplete_update_holds_last_committed_projection_and_exposes_retry(portal, world, monkeypatch): + fake, signer = world + fake.request("saw-req-initial", ds_request(signer)) + assert portal.main(["create", "saw-req-initial"]) == 0 + previous = portal.read_entry(fake.objects[REGISTRY]) + update = {**previous, "profiles": ["different"]} # only public registration data + pending = portal.reserve_credentials(portal.kube(), "saw-portal", "saw-ws-alice", fake.objects[REGISTRY], update, "alice") + assert portal.list_workspaces(portal.kube(), "saw-portal") == [previous] + assert portal.list_workspaces(portal.kube(), "saw-portal", deleting=True) == [update] + holder = pending["metadata"]["annotations"][portal.CREDENTIAL_WRITER] + run_path = "/apis/tekton.dev/v1/namespaces/saw-portal/pipelineruns/" + holder + for run, phase in [({}, "failed"), ({"metadata":{"name":holder}}, "publishing"), + ({"status":{"conditions":[{"type":"Succeeded","status":"False"}]}}, "failed")]: + if run: + fake.objects[run_path] = run + assert portal.workspace_status(portal.kube(), "alice", {}, "saw-portal", "argo", "")["phase"] == phase + retry = portal.reserve_credentials(portal.kube(), "saw-portal", "saw-ws-alice", pending, update, "alice") + assert portal.committed_entry(retry) == previous + + +def test_catalog_probe_is_async_bounded_and_scoped_to_registration(portal): + cache = portal.CatalogRouteCache(workers=2, capacity=4, ttl=30) + release = threading.Event() + calls = [] + def slow(url): + calls.append(url) + return release.wait(5) + try: + for n in range(40): + assert not cache.probe(("owner","registration"), f"https://route-{n}", slow) + assert len(cache.entries) == 4 + release.set() + for future in cache.entries.values(): + assert future.result(timeout=5)[1] + assert cache.probe(("owner","registration"), "https://route-0", slow) + assert not cache.probe(("owner","new-registration"), "https://route-0", slow) + finally: + release.set() + cache.pool.shutdown() + + +def test_dedicated_endpoint_excludes_renamed_and_additional_tool_secrets(portal, monkeypatch): + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","container"]') + profile = {"workspaces": [], "dedicatedEndpointSecrets":["model"], + "secrets":{key:{"provider":"test","providers":["test"],"fields":[ + {"key":"api_key","kind":"secret","required":True}]} + for key in ("model","search-renamed","another-tool")}} + data = {"profile":"p","deploymentMode":"container","model.api_key":"fixture"} + _, values = portal.parse_request(data, {"p":profile}) + assert set(values) == {"model"} + with pytest.raises(portal.PortalError, match="search-renamed"): + portal.parse_request({**data,"deploymentMode":"vm"}, {"p":profile}) + + +@pytest.mark.parametrize("mode,stage,expected", [ + ("container", "vm", 1800), ("vm", "vm", 600), ("container", "running", 600) +]) +def test_enrollment_budget_only_extends_the_container_intent_wait(portal, world, monkeypatch, mode, stage, expected): + created(portal, world) + monkeypatch.setenv("SAW_CONTAINER_ENROLLMENT_SECONDS", "1800") + monkeypatch.setattr(portal, "workspace_status", lambda *args: { + "deploymentMode": mode, "failed": False, "message": "pending", + "steps": [{"id": stage, "state": "active", "detail": "pending"}], + }) + now = [0] + with pytest.raises(portal.PortalError, match=f"not done after {expected // 60} minutes"): + portal.wait_stage(stage, "alice", 600, sleep=lambda _: now.__setitem__(0, now[0] + 600), clock=lambda: now[0]) + assert now == [expected] + + +@pytest.mark.parametrize("requested, bound", [("kata", "container"), ("container", "kata")]) +def test_portal_rejects_wrong_site_placement_before_registration(portal, world, monkeypatch, requested, bound): + fake, signer = world + subject = "478fdd1f-8db9-41e8-97ad-b563215c89e9" + site = {"subject": subject, "automaticEnrollment": {"approvedProfiles": {"audited-openclaw": "a"*64}}} + if bound == "kata": + site["kataRuntimeClass"] = {"name": "saw-kata", "uid": subject, "sha256": "b"*64} + monkeypatch.setenv("SAW_DEPLOYMENT_MODES", '["vm","container","kata"]') + monkeypatch.setenv("SAW_CONTAINER_SITES", json.dumps({"alice": site})) + fake.request("saw-req-placement", ds_request(signer, deploymentMode=requested, profile="audited-openclaw")) + assert portal.main(["create", "saw-req-placement"]) != 0 + assert not fake.vault and REGISTRY not in fake.objects diff --git a/tests/provider-inference-smoke.mjs b/tests/provider-inference-smoke.mjs new file mode 100644 index 000000000..c15c8643b --- /dev/null +++ b/tests/provider-inference-smoke.mjs @@ -0,0 +1,46 @@ +// Run inside a sandbox with its injected placeholder and normal proxy path. +// Never run this with a real provider key in the sandbox environment. +import {randomUUID} from 'node:crypto'; + +const [kind, model] = process.argv.slice(1); +const providers = { + openai: {url:'https://api.openai.com/v1/chat/completions', env:'OPENAI_API_KEY'}, + anthropic: {url:'https://api.anthropic.com/v1/messages', env:'ANTHROPIC_API_KEY'}, + nvidia: {url:'https://integrate.api.nvidia.com/v1/chat/completions', env:'NVIDIA_API_KEY'}, +}; +const provider = providers[kind]; +if (!provider || !/^[A-Za-z0-9][A-Za-z0-9._/-]{0,180}$/.test(model ?? '')) { + console.error('expected provider (openai|anthropic|nvidia) and approved model'); + process.exit(2); +} +if (!process.env[provider.env]) { + console.error('sandbox provider placeholder is missing'); + process.exit(2); +} +const nonce = 'SAW_PROVIDER_' + randomUUID().replaceAll('-', ''); +const headers = {'content-type':'application/json'}; +if (kind === 'anthropic') { + headers['x-api-key'] = process.env[provider.env]; + headers['anthropic-version'] = '2023-06-01'; +} else { + headers.authorization = 'Bearer ' + process.env[provider.env]; +} +const body = {model, messages:[{role:'user',content:'Reply with exactly this token and nothing else: '+nonce}]}; +body[kind === 'openai' ? 'max_completion_tokens' : 'max_tokens'] = 128; +try { + const response = await fetch(provider.url, { + method:'POST', headers, body:JSON.stringify(body), signal:AbortSignal.timeout(90000), + redirect:'error', + }); + // Provider error bodies and headers can contain private account details. + if (!response.ok) throw new Error('provider HTTP '+response.status); + const result = await response.json(); + const text = kind === 'anthropic' ? result.content?.filter(c=>c.type==='text').map(c=>c.text).join('') + : result.choices?.[0]?.message?.content; + const matched = typeof text === 'string' && text.trim() === nonce; + console.log(JSON.stringify({provider:kind, model, httpStatus:response.status, nonceMatched:matched})); + if (!matched) process.exitCode = 1; +} catch (error) { + console.error(error.message?.startsWith('provider HTTP ') ? error.message : 'provider request failed'); + process.exitCode = 1; +} diff --git a/tests/scripts/test_container_installation.py b/tests/scripts/test_container_installation.py new file mode 100644 index 000000000..31e55ec10 --- /dev/null +++ b/tests/scripts/test_container_installation.py @@ -0,0 +1,348 @@ +"""Standalone entry point: public inputs, ownership and shared lifecycle.""" +import copy +import importlib.util +import json +from pathlib import Path +import sys + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location("container_installation", ROOT / "scripts/container-workspace.py") +install = importlib.util.module_from_spec(spec) +spec.loader.exec_module(install) + + +@pytest.fixture(autouse=True) +def standalone_container_mode(monkeypatch): + # The documented container invocation sets this explicitly. The outer + # make test-installer invocation can export its own VM default. + monkeypatch.setenv("DEPLOYMENT_MODE", "container") + + +def inputs(): + site = yaml.safe_load((ROOT / "tests/fixtures/container-site.yaml").read_text())["portal"]["containerSites"]["alice"] + user = dict(name="alice", registration="a"*16, installationID="22222222-2222-4222-8222-222222222222", + ownerSubject=site["subject"], profiles=["data-science"], intentGeneration=1, + vaultPrefix="secret/data/hub/saw-alice/"+"a"*16, pruneOnRemove=True, + values={"deployment": {"mode": "container"}}) + return {"users": [user, {"name": "bob"}], "containerSites": {"alice": site}, + "ownerResolution": {"enabled": True}, + "global": {"vpArgoNamespace": "vp-gitops", "clusterDomain": "cluster.example.test"}} + + +def test_select_isolates_container_without_modifying_vm_or_input(): + values = inputs() + before = copy.deepcopy(values) + selected, user = install.select(values, "alice") + assert values == before + assert selected["users"] == [user] + assert selected["ownerResolution"]["enabled"] is False + assert selected["projection"]["enabled"] is True + + +@pytest.mark.parametrize(("requested", "declared"), [ + ("container", "kata"), ("kata", "container"), ("vm", "container"), +]) +def test_requested_backend_must_match_the_registration(monkeypatch, requested, declared): + monkeypatch.setenv("DEPLOYMENT_MODE", requested) + values = inputs() + values["users"][0]["values"]["deployment"]["mode"] = declared + with pytest.raises(install.InstallError, match="requested Kubernetes deployment mode"): + install.select(values, "alice") + + +@pytest.mark.parametrize("fault", ["missing", "duplicate", "mode", "subject", "registration", "generation", "cleanup"]) +def test_invalid_input_cannot_start_installation(fault): + values = inputs() + user = values["users"][0] + if fault == "missing": values["users"] = [] + if fault == "duplicate": values["users"].append(copy.deepcopy(user)) + if fault == "mode": user["values"]["deployment"]["mode"] = "vm" + if fault == "subject": user["ownerSubject"] = "33333333-3333-4333-8333-333333333333" + if fault == "registration": user["registration"] = "unbound" + if fault == "generation": user["intentGeneration"] = True + if fault == "cleanup": user["pruneOnRemove"] = False + with pytest.raises(install.InstallError): install.select(values, "alice") + + +@pytest.mark.parametrize("field", ["registration", "installationID", "ownerSubject", "name"]) +def test_update_cannot_adopt_another_registration(field): + values, user = install.select(inputs(), "alice") + existing = copy.deepcopy(values) + existing["users"][0][field] = "different" + with pytest.raises(install.InstallError, match="refusing adoption"): + install.check_existing(existing, user) + install.check_existing(values, user) + + +def test_make_preflight_accepts_a_first_registration_with_complete_approval(monkeypatch): + values, user = install.select(inputs(), "alice") + del values["containerSites"]["alice"]["providerPolicy"] + assert install.preflight_checks.configuration_facts(values["containerSites"]["alice"])["providerEgress"] + calls = [] + monkeypatch.setattr(install, "get", lambda kind, name, namespace=None: + calls.append((kind, name)) or None) + install.check_live_boundary_approval(values["containerSites"]["alice"], "container", user) + assert calls == [("namespace", "saw-alice")] + + +def test_make_preflight_rejects_nontraversable_image_before_cluster_reads(monkeypatch): + values, user = install.select(inputs(), "alice") + del values["containerSites"]["alice"]["profileImageOverrides"]["data-science"]["notebook"] + calls = [] + monkeypatch.setattr(install, "get", lambda *args: calls.append(args)) + with pytest.raises(install.InstallError, match="WorkloadImageNotNamespaceAccessible"): + install.preflight(values, "workspaces", user) + assert calls == [] + + +def test_make_preflight_rejects_a_stale_live_boundary_before_apply(monkeypatch): + values, user = install.select(inputs(), "alice") + site = values["containerSites"]["alice"] + owner, registration, installation = ( + user["name"], user["registration"], user["installationID"]) + policy = "saw-saw-alice-runtime" + catalog = json.loads((ROOT / "charts/saw-users/files/profile-catalog.json").read_text())["profiles"] + slots = sorted(install.workspace.sandbox_targets( + owner, registration, catalog["data-science"])) + relays = [install.preflight_checks.hashlib.sha256((registration + "/" + slot).encode()).hexdigest()[:16] + if len(slots) > 1 else registration for slot in slots] + validation_spec = { + "controlNamespace": "saw-ctl-" + registration, + "installation": installation, + "mode": "container", + "gatewaySlots": [owner + "-" + relay[:8] for relay in relays] if len(slots) > 1 else [], + "gateRelays": ["gate-" + relay + "-claims" for relay in relays], + "browserProofs": ["ui-" + relay + "-browser-proof" for relay in relays], + } + objects = { + ("namespace", "saw-alice"): {"metadata": {"labels": { + "saw.redhat.com/registration": registration, + "saw.redhat.com/runtime-owner": installation, + }}}, + ("validatingadmissionpolicies.admissionregistration.k8s.io", policy): { + "kind": "ValidatingAdmissionPolicy", "metadata": {"generation": 1}, + "spec": validation_spec, + "status": {"observedGeneration": 1, "typeChecking": {"expressionWarnings": []}}, + }, + } + monkeypatch.setattr(install, "get", lambda kind, name, namespace=None: objects.get((kind, name))) + with pytest.raises(install.InstallError, match="BoundaryApprovalChanged"): + install.check_live_boundary_approval(site, "container", user) + + +def test_preflight_fails_without_required_api_and_never_installs_dependency(monkeypatch): + config = inputs() + user = install.select(config, "alice")[1] + calls = [] + def get(kind, name, namespace=None): + calls.append((kind, name)) + if kind == "namespace": + return None + if kind == "clusterversion": + return {"status": {"conditions": [{"type": "Available", "status": "True"}], + "desired": {"version": "4.22.18"}}} + if kind == "crd": + if name == "sandboxes.agents.x-k8s.io": + return None + return {"status": {"conditions": [{"type": "Established", "status": "True"}]}} + if kind == "clustersecretstore": + return {"metadata": {"name": name}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}} + raise AssertionError(f"unexpected read: {kind} {name}") + + run_calls = [] + def run(*args, **kwargs): + run_calls.append(args) + if args == ("oc", "get", "--raw", "/apis/admissionregistration.k8s.io/v1"): + return json.dumps({ + "groupVersion": "admissionregistration.k8s.io/v1", + "resources": [{"name": "validatingadmissionpolicies"}, + {"name": "validatingadmissionpolicybindings"}], + }) + if args[:3] == ("oc", "get", "storageclasses"): + return json.dumps({"items": [ + {"metadata": {"name": "block"}}, + {"metadata": {"name": "block-immediate"}}, + {"metadata": {"name": "shared-filesystem"}}, + ]}) + if args[:3] == ("oc", "auth", "can-i"): + return "yes" + raise AssertionError(f"unexpected command: {args}") + + monkeypatch.setattr(install, "get", get) + monkeypatch.setattr(install, "run", run) + with pytest.raises(install.InstallError, match="AgentSandboxUnavailable"): + install.preflight(config, "workspaces", user) + assert ("crd", "sandboxes.agents.x-k8s.io") in calls + assert all(args[:2] not in (("oc", "apply"), ("oc", "create"), ("oc", "delete")) + for args in run_calls) + + +def test_namespace_create_is_explicit_and_never_patches_existing_namespace(monkeypatch): + calls = [] + monkeypatch.setattr(install, "run", lambda *args, **kwargs: calls.append((args, kwargs))) + install.ensure_namespace({}, "workspaces", "vp-gitops") + assert calls[0][0] == ("oc", "create", "-f", "-") + body = json.loads(calls[0][1]["input_text"]) + assert body["metadata"]["labels"]["argocd.argoproj.io/managed-by"] == "vp-gitops" + install.ensure_namespace(body, "workspaces", "vp-gitops") + assert len(calls) == 1 + + +def test_status_uses_shared_readiness_and_never_reads_secrets(monkeypatch): + values, user = install.select(inputs(), "alice") + observations = [] + monkeypatch.setattr(install, "get", lambda *args: observations.append(args)) + def check(read, owner, site, argo, domain, ping, catalog): + assert owner == user and site["subject"] == user["ownerSubject"] + assert argo == "vp-gitops" and domain == "cluster.example.test" + read("/apis/argoproj.io/v1alpha1/namespaces/vp-gitops/applications/saw-alice") + read("/api/v1/namespaces/saw-alice/configmaps/saw-runtime-status") + assert callable(ping) and "data-science" in catalog + return {"ready": False} + monkeypatch.setattr(install.workspace, "container_status", check) + assert install.status(user, values) == {"ready": False} + assert observations == [("application", "saw-alice", "vp-gitops"), + ("configmap", "saw-runtime-status", "saw-alice")] + + +@pytest.mark.parametrize("action", ["apply", "status", "delete"]) +def test_commands_use_only_owned_helm_release_and_public_observations(action, monkeypatch, tmp_path): + values, user = install.select(inputs(), "alice") + filename = tmp_path / "values.yaml" + filename.write_text(yaml.safe_dump(values)) + calls = [] + def run(*args, **kwargs): + calls.append(args) + if args[:2] == ("helm", "list"): return json.dumps([{"name": "workspace-alice"}]) + if args[:3] == ("helm", "get", "values"): return json.dumps(values) + return "" + monkeypatch.setattr(install, "run", run) + monkeypatch.setattr(install.shutil, "which", lambda _: "/tool") + def get(kind, name, namespace=None): + if kind == "ingress.config.openshift.io": return {"spec": {"domain": "apps.cluster.example.test"}} + return {"metadata": {"labels": {"saw.redhat.com/runtime-owner": user["installationID"]}}} + monkeypatch.setattr(install, "get", get) + monkeypatch.setattr(install, "status", lambda *_: {"ready": True}) + monkeypatch.setattr(install, "preflight", lambda *_: ({"metadata": {}}, "vp-gitops")) + monkeypatch.setattr(sys, "argv", ["container-workspace.py", action, "--name", "alice", "--values", str(filename)]) + assert install.main() == 0 + writes = [args for args in calls if args[:2] in (("helm", "upgrade"), ("helm", "uninstall"))] + assert len(writes) == (action != "status") + if action != "status": assert "workspace-alice" in writes[0] + if action == "delete": assert writes[0][-5:] == ("--cascade", "foreground", "--wait", "--timeout", "5m") + assert not any("secret" in args or "patch" in args or "delete" in args for args in calls) + + +def test_server_dry_run_separates_new_and_helm_owned_resources(monkeypatch): + resources = [ + {"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "new"}}, + {"apiVersion": "admissionregistration.k8s.io/v1", "kind": "ValidatingAdmissionPolicy", + "metadata": {"name": "owned"}, "spec": {"matchConstraints": {"resourceRules": []}}}, + ] + commands = [] + def run(*args, **kwargs): + commands.append((args, kwargs)) + if args[:3] == ("oc", "get", "ConfigMap"): + return "" + if args[:3] == ("oc", "get", "ValidatingAdmissionPolicy"): + return "workspace-alice|workspaces|Helm" + return "" + monkeypatch.setattr(install, "run", run) + install.dry_run_rendered(yaml.safe_dump_all(resources), "workspaces", "workspace-alice") + applies = [(args, kwargs) for args, kwargs in commands if args[:2] == ("oc", "apply")] + assert len(applies) == 2 + assert "--force-conflicts" not in applies[0][0] + assert "--force-conflicts" in applies[1][0] + assert all("--dry-run=server" in args for args, _ in applies) + assert [item["metadata"]["name"] for item in yaml.safe_load_all(applies[0][1]["input_text"])] == ["new"] + assert [item["metadata"]["name"] for item in yaml.safe_load_all(applies[1][1]["input_text"])] == ["owned"] + + +def test_server_dry_run_rejects_foreign_resource_before_apply(monkeypatch): + resource = {"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "foreign"}} + calls = [] + def run(*args, **kwargs): + calls.append(args) + return "another-release|workspaces|Helm" + monkeypatch.setattr(install, "run", run) + with pytest.raises(install.InstallError, match="outside Helm release"): + install.dry_run_rendered(yaml.safe_dump(resource), "workspaces", "workspace-alice") + assert all(args[:2] != ("oc", "apply") for args in calls) + + +def test_server_dry_run_preserves_admission_rejection(monkeypatch): + resource = {"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "new"}} + def run(*args, **kwargs): + if args[:2] == ("oc", "apply"): + raise install.InstallError("OpenShift admission rejected the resource") + return "" + monkeypatch.setattr(install, "run", run) + with pytest.raises(install.InstallError, match="admission rejected"): + install.dry_run_rendered(yaml.safe_dump(resource), "workspaces", "workspace-alice") + + +@pytest.mark.parametrize(("error", "expected"), [ + ("Unable to connect to the server: dial tcp: lookup example: no such host", "DNS lookup failed"), + ("Error from server (Forbidden): access denied", "HTTP 403"), + ('Error from server (Conflict): conflict with "helm"', "field ownership conflict"), + ("unexpected response", "oc operation failed"), +]) +def test_oc_diagnostic_does_not_echo_stderr(monkeypatch, error, expected): + def failed(*args, **kwargs): + return install.subprocess.CompletedProcess(args, 1, "", error + " private-fixture-value") + monkeypatch.setattr(install.subprocess, "run", failed) + with pytest.raises(install.InstallError, match=expected) as exc: + install.run("oc", "get", "namespace", "test") + assert "private-fixture-value" not in str(exc.value) + + +def test_make_applies_checked_overlay_before_render_and_fails_before_mutation_on_drift(monkeypatch, tmp_path): + values, user = install.select(inputs(), "alice") + values["containerSites"]["alice"]["automaticEnrollment"]["enabled"] = False + base = tmp_path / "site.yaml" + overlay = tmp_path / "net.yaml" + base.write_text(yaml.safe_dump(values)) + network = install.container_topology + site = values["containerSites"]["alice"] + ip = ["10.0.0.7"] + def discover(config, probe, container): + assert probe == "ops/probe" and config["containerSites"]["alice"] == site + result = {"projection": {"egress": []}, "containerSites": {"alice": {}}} + o = result["containerSites"]["alice"] + for path in (network.GATEWAY + ("oidc", "issuerUrl"), network.RELAY + ("issuer",)): + network.put(o, path, "https://issuer.example") + for path in (network.BOOTSTRAP, network.RELAY + ("egress",), network.GATEWAY + ("kubernetes", "network", "gatewayEgress")): + network.put(o, path, [network.host_rule(ip[0], 443)]) + return result + + monkeypatch.setenv("SAW_TOPOLOGY_OVERLAY", str(overlay)) + monkeypatch.setenv("SAW_TOPOLOGY_PROBE", "ops/probe") + monkeypatch.setattr(network, "container_overlay", discover) + monkeypatch.setattr(sys, "argv", ["container-workspace.py", "apply", "--name", "alice", "--values", str(base)]) + monkeypatch.setattr(install.shutil, "which", lambda _: "/tool") + monkeypatch.setattr(install, "get", lambda kind, *a: {"spec":{"domain":"apps.cluster.example.test"}} + if kind == "ingress.config.openshift.io" else None) + monkeypatch.setattr(install, "preflight", lambda *args: ({}, "vp-gitops")) + writes = [] + monkeypatch.setattr(install, "ensure_namespace", lambda *args: writes.append("namespace")) + def run(*args, **kwargs): + if args[:2] == ("helm", "list"): + return "[]" + if args[:2] in (("helm", "template"), ("helm", "upgrade")): + rendered_values = json.loads(Path(args[args.index("-f")+1]).read_text()) + assert network.path_value(rendered_values["containerSites"]["alice"], network.RELAY + ("egress",)) == [ + network.host_rule("10.0.0.7", 443)] + if args[:2] == ("helm", "upgrade"): + writes.append("helm") + return "" + monkeypatch.setattr(install, "run", run) + assert install.main() == 0 and writes == ["namespace", "helm"] + ip[0] = "10.0.0.8" + writes.clear() + with pytest.raises(SystemExit): + install.main() + assert not writes diff --git a/tests/scripts/test_container_topology.py b/tests/scripts/test_container_topology.py new file mode 100644 index 000000000..60dc3985d --- /dev/null +++ b/tests/scripts/test_container_topology.py @@ -0,0 +1,270 @@ +"""Role-scoped topology, review protection, and the in-Pod HTTPS check.""" +import copy +import importlib.util +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] + + +def module(name): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / (name + ".py")) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +topo = module("container_topology") +probe = module("oidc_topology_probe") + + +@pytest.fixture +def cluster(monkeypatch): + calls = [] + addresses = ["10.10.10.101", "10.10.10.102"] + def command(*args): + calls.append(args) + if "print-json" in args: + return json.dumps({"cleanupApi": {"serviceIP": "172.30.0.1", "endpoints": [ + {"ip": "10.10.1.2", "port": 6443}]}}) + if "dns-default" in args: + return json.dumps({"spec": {"clusterIP": "172.30.0.10", + "selector": {"dns.operator.openshift.io/daemonset-dns": "default"}, + "ports": [{"name": p.lower(), "port": 53, "protocol": p} for p in ["TCP", "UDP"]]}}) + if "endpointslice" in args: + return json.dumps({"items": [{"addressType": "IPv4", "ports": [ + {"name": p.lower(), "port": 5353, "protocol": p} for p in ["UDP", "TCP"]]}]}) + if "ingress.config.openshift.io" in args: + return '{"spec":{"domain":"apps.cluster.example.test"}}' + if args[0].endswith("keycloak-host.sh"): + return "keycloak.apps.cluster.example.test" + if "exec" in args: + assert args[1:5] == ("-n", "trusted-tools", "exec", "pod/network-probe") + return json.dumps(addresses) + pytest.fail(f"unexpected command: {args[:3]}") + monkeypatch.setattr(topo, "command", command) + return addresses, calls + + +@pytest.fixture +def base(): + site = yaml.safe_load((ROOT / "tests/fixtures/container-site.yaml").read_text())["portal"]["containerSites"]["alice"] + site["automaticEnrollment"]["enabled"] = False + for prefix, field in [(topo.GATEWAY + ("kubernetes", "network"), "gatewayEgress"), + (topo.BOOTSTRAP[:-1], "bootstrapEgress"), (topo.RELAY, "egress")]: + topo.path_value(site, prefix)[field] = [] + return {"containerSites": {"alice": site}, "global": {"clusterDomain": "cluster.example.test"}} + + +def test_overlay_uses_one_probe_and_preserves_separation(base, cluster): + addresses, calls = cluster + site = base["containerSites"]["alice"] + operator_rule = {"to": [{"podSelector": {"matchLabels": {"role": "proxy"}}}], + "ports": [{"port": 3128, "protocol": "TCP"}]} + topo.path_value(site, topo.GATEWAY + ("kubernetes", "network", "gatewayEgress")).append(operator_rule) + base["portal"] = {"containerSites": {"alice": copy.deepcopy(site)}} + before = copy.deepcopy(base) + overlay = topo.container_overlay(base, "trusted-tools/network-probe", "tools") + assert base == before + assert len([c for c in calls if "exec" in c]) == 1 + assert overlay["portal"]["containerSites"] == overlay["containerSites"] + result = overlay["containerSites"]["alice"] + gateway = topo.path_value(result, topo.GATEWAY + ("kubernetes", "network", "gatewayEgress")) + relay = topo.path_value(result, topo.RELAY + ("egress",)) + bootstrap = topo.path_value(result, topo.BOOTSTRAP) + assert gateway == [operator_rule, *relay] + assert relay == bootstrap + [topo.host_rule(ip, 443) for ip in addresses] + assert bootstrap == overlay["projection"]["egress"] + assert topo.host_rule("172.30.0.1", 443) in bootstrap + assert topo.host_rule("10.10.1.2", 6443) in bootstrap + assert topo.host_rule("10.10.1.2", 443) not in bootstrap + assert {"to": [{"ipBlock": {"cidr": "172.30.0.10/32"}}], + "ports": [{"protocol": "TCP", "port": 53}, {"protocol": "UDP", "port": 53}]} in bootstrap + assert set(result["stack"]) == {"gateway", "proxy"} + assert set(result["stack"]["proxy"]["values"]["network"]) == {"bootstrapEgress"} + + +def test_make_bootstrap_saves_once_enrolled_apply_rechecks_and_delete_works_offline(base, cluster, tmp_path): + values = copy.deepcopy(base) + file = tmp_path / "net.yaml" + topo.install_values(values, "alice", file, "trusted-tools/network-probe", None, verify=True) + approved = file.read_bytes() + cluster[0].append("10.10.10.103") + with pytest.raises(ValueError, match="stale"): + topo.install_values(copy.deepcopy(base), "alice", file, "trusted-tools/network-probe", None, verify=True) + assert file.read_bytes() == approved + calls = len(cluster[1]) + topo.install_values(copy.deepcopy(base), "alice", file, None, None, verify=False) + assert len(cluster[1]) == calls + base["containerSites"]["alice"]["automaticEnrollment"]["enabled"] = True + with pytest.raises(ValueError, match="review"): + topo.install_values(base, "alice", tmp_path/"missing", "trusted-tools/network-probe", None, verify=True) + assert not (tmp_path/"missing").exists() + + +def test_default_and_conflicting_issuer(base, cluster): + site = base["containerSites"]["alice"] + del topo.path_value(site, topo.GATEWAY + ("oidc",))["issuerUrl"] + del topo.path_value(site, topo.RELAY)["issuer"] + result = topo.container_overlay(base, "trusted-tools/network-probe") + assert topo.path_value(result["containerSites"]["alice"], topo.RELAY + ("issuer",)) == ( + "https://keycloak.apps.cluster.example.test/realms/openshell") + topo.put(site, topo.RELAY + ("issuer",), "https://a.example/realm") + topo.put(site, topo.GATEWAY + ("oidc", "issuerUrl"), "https://b.example/realm") + with pytest.raises(ValueError, match="same issuer"): + topo.container_overlay(base, "trusted-tools/network-probe") + + +@pytest.mark.parametrize("url", ["http://issuer.example", "https://me@issuer.example", "https://issuer.example?q=1", + "https://issuer.example/#here", "https://issuer.example:0", + "https://issuer.example?", "https://issuer.example/#", "https://issuer.example\\path"]) +def test_unsafe_issuer_rejected_before_exec(url, cluster): + with pytest.raises(ValueError): + topo.discover_issuer(url, "trusted-tools/network-probe", None) + assert not cluster[1] + + +@pytest.mark.parametrize(("status", "claimed", "passes"), [(200, "https://k.test:8443/realm", True), + (302, "https://k.test:8443/realm", False), (200, "https://different.test/realm", False)]) +def test_each_address_uses_original_tls_name_and_checks_document(monkeypatch, status, claimed, passes): + hosts = ["10.0.0.50", "10.0.0.51"] + tls = MagicMock() + monkeypatch.setattr(probe.ssl, "create_default_context", lambda: tls) + monkeypatch.setattr(probe.socket, "getaddrinfo", lambda *a, **kw: [(None, None, None, None, (ip, 8443)) for ip in hosts]) + connect = MagicMock() + monkeypatch.setattr(probe.socket, "create_connection", connect) + https = MagicMock() + monkeypatch.setattr(probe.http.client, "HTTPSConnection", https) + response = https.return_value.getresponse.return_value + response.status = status + response.read.return_value = json.dumps({"issuer": claimed}).encode() + issuer = "https://k.test:8443/realm" + if not passes: + with pytest.raises(ValueError, match="mismatch"): + probe.probe(issuer) + return + assert probe.probe(issuer) == hosts + assert [c.args[0] for c in connect.call_args_list] == [(ip, 8443) for ip in hosts] + assert all(c.kwargs["server_hostname"] == "k.test" for c in tls.wrap_socket.call_args_list) + assert all(c.args == ("GET", "/realm/.well-known/openid-configuration") for c in https.return_value.request.call_args_list) + + +@pytest.mark.parametrize("bad", ["127.0.0.1", "169.254.169.254", "224.0.0.1", "::1"]) +def test_probe_refuses_unsafe_or_unhandled_answers(monkeypatch, bad): + monkeypatch.setattr(probe.socket, "getaddrinfo", lambda *a, **kw: [(None, None, None, None, (bad, 443))]) + connect = MagicMock() + monkeypatch.setattr(probe.socket, "create_connection", connect) + with pytest.raises(ValueError, match="unsafe"): + probe.probe("https://keycloak.example/realms/openshell") + connect.assert_not_called() + + +@pytest.mark.parametrize("hostname", ["issuer.example", "wrong.example"]) +def test_probe_verifies_certificate_over_real_tls(tmp_path, monkeypatch, hostname): + import socket + import ssl + import subprocess + from concurrent.futures import ThreadPoolExecutor + from threading import Event + + cert, key = tmp_path / "cert.pem", tmp_path / "key.pem" + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=issuer.example", "-addext", "subjectAltName=DNS:issuer.example", + "-keyout", str(key), "-out", str(cert)], check=True, capture_output=True) + server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server.load_cert_chain(cert, key) + monkeypatch.setenv("SSL_CERT_FILE", str(cert)) + issuer = f"https://{hostname}/realms/test" + response_read = Event() + client_sock, server_sock = socket.socketpair() + for sock in (client_sock, server_sock): + sock.settimeout(5) + monkeypatch.setattr(probe.socket, "getaddrinfo", + lambda *args, **kwargs: [(None, None, None, None, ("10.0.1.2", 443))]) + monkeypatch.setattr(probe.socket, "create_connection", lambda *args, **kwargs: client_sock) + + def serve(): + with server.wrap_socket(server_sock, server_side=True) as stream: + with stream.makefile("rb") as request: + assert request.readline() == b"GET /realms/test/.well-known/openid-configuration HTTP/1.1\r\n" + headers = [] + while (line := request.readline()) not in (b"\r\n", b""): + headers.append(line) + assert b"Host: issuer.example\r\n" in headers + payload = json.dumps({"issuer": issuer}).encode() + stream.sendall(f"HTTP/1.1 200 OK\r\nContent-Length: {len(payload)}\r\n\r\n".encode() + payload) + assert response_read.wait(timeout=5) + + with client_sock, server_sock, ThreadPoolExecutor(max_workers=1) as pool: + served = pool.submit(serve) + if hostname == "issuer.example": + try: + assert probe.probe(issuer) == ["10.0.1.2"] + finally: + response_read.set() + served.result(timeout=5) + else: + with pytest.raises(ssl.SSLCertVerificationError): + probe.probe(issuer) + with pytest.raises(ssl.SSLError): + served.result(timeout=5) + + +@pytest.mark.ci_external +def test_make_commands_and_helm_gateway_policy(tmp_path, monkeypatch, base): + import subprocess + import os + import sys + + # The synthetic oc is only a cluster fixture. Exercise the real VM discovery + # script, Make entry points, YAML review comparison and stock Helm template. + oc = tmp_path / "oc" + oc.write_text("""#!/usr/bin/env python3 +import sys, json, os +args = sys.argv[1:] +if "exec" in args: + data = [os.environ["MOCK_ISSUER_IP"]] +elif "network.config" in args: + data = {"status":{"clusterNetwork":[{"cidr":"10.128.0.0/14"}],"serviceNetwork":["172.30.0.0/16"]}} +elif "ingress.config.openshift.io" in args: + data = {"spec":{"domain":"apps.cluster.example.test"}} +elif "service" in args and "kubernetes" in args: + data = {"spec":{"clusterIP":"172.30.0.1","ports":[{"port":443}]}} +elif "endpointslice" in args and "default" in args: + data = {"items":[{"addressType":"IPv4","ports":[{"port":6443}],"endpoints":[{"addresses":["10.0.0.3"]}]}]} +elif "service" in args and "dns-default" in args: + data = {"spec":{"clusterIP":"172.30.0.10","selector":{"dns.operator.openshift.io/daemonset-dns":"default"}, + "ports":[{"name":p.lower(),"port":53,"protocol":p} for p in ["UDP","TCP"]]}} +elif "endpointslice" in args and "openshift-dns" in args: + data = {"items":[{"addressType":"IPv4","ports":[{"name":p.lower(),"port":5353,"protocol":p} for p in ["UDP","TCP"]]}]} +else: + sys.exit(3) +print(json.dumps(data)) +""") + oc.chmod(0o755) + monkeypatch.setenv("PATH", str(tmp_path) + os.pathsep + os.environ["PATH"]) + monkeypatch.setenv("MOCK_ISSUER_IP", "10.10.10.101") + base_file = tmp_path / "site.yaml" + out = tmp_path / "overlay.yaml" + base_file.write_text(yaml.safe_dump(base)) + monkeypatch.setenv("SAW_USERS_VALUES_FILE", str(base_file)) + monkeypatch.setenv("SAW_TOPOLOGY_OVERLAY", str(out)) + monkeypatch.setenv("SAW_TOPOLOGY_PROBE", "trusted-tools/network-probe") + make = ["make", "--no-print-directory", "PYTHON=" + sys.executable] + out.write_text(subprocess.check_output([*make, "container-topology"], text=True, cwd=ROOT)) + subprocess.run([*make, "check-container-topology"], check=True, cwd=ROOT) + gateway = yaml.safe_load(out.read_text())["containerSites"]["alice"]["stack"]["gateway"]["values"] + rendered = subprocess.check_output(["helm", "template", "saw-test", "charts/openshell-saw-kubernetes", + "-n", "saw-test", "--api-versions", "agents.x-k8s.io/v1beta1/Sandbox", + "--api-versions", "policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy", "-f", "tests/fixtures/saw-kubernetes-values.yaml", + "-f", "-"], input=yaml.safe_dump(gateway), text=True, cwd=ROOT) + policies = [r for r in yaml.safe_load_all(rendered) if r and r["kind"] == "NetworkPolicy"] + matching = [r for r in policies if topo.host_rule("10.10.10.101", 443) in r["spec"].get("egress", [])] + assert len(matching) == 1 and matching[0]["spec"]["podSelector"]["matchLabels"]["openshell.pattern/role"] == "gateway" + monkeypatch.setenv("MOCK_ISSUER_IP", "10.10.10.111") + drift = subprocess.run([*make, "check-container-topology"], text=True, cwd=ROOT, capture_output=True) + assert drift.returncode and "topology drift" in drift.stderr diff --git a/tests/scripts/test_deployment_validation_ci.py b/tests/scripts/test_deployment_validation_ci.py index 42189c9e1..9ceb1d069 100644 --- a/tests/scripts/test_deployment_validation_ci.py +++ b/tests/scripts/test_deployment_validation_ci.py @@ -3,14 +3,30 @@ import json import io import sys +import shutil from pathlib import Path from types import SimpleNamespace +import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from scripts.ci import deployment_validation as dv from scripts.ci.vendor_crd_schemas import validation_only +@pytest.mark.skipif(not shutil.which("helm"), reason="Helm is required") +def test_enabled_chart_graph_renders_every_component(tmp_path): + result = dv.Validation() + dv.render_charts(result, tmp_path) + assert result.errors == [] + for chart in dv.charts(): + name = chart.parent.relative_to(dv.ROOT).as_posix().replace("/", "_") + documents = list(dv.yaml.safe_load_all((tmp_path / (name + ".yaml")).read_text())) + assert any(isinstance(item, dict) and item.get("kind") for item in documents) + controller = dv.yaml.safe_load((tmp_path / "saw-c-a8cb6cbd0eb62f31-gateway-inputs.json").read_text()) + assert controller["kubernetes"]["experimental"] is True + assert controller["kubernetes"]["identity"]["clientCaSecret"] == "alice-client-ca" + + def test_vendored_schemas_have_no_upstream_sample_data(): source = {"properties": {"password": {"type": "string", "example": "sample"}}, "description": "sample", "default": "sample"} @@ -26,6 +42,7 @@ def test_vendored_crd_schema_names_match_kubeconform_lookup(): ("hco.kubevirt.io", "hyperconverged_v1beta1.json"), ("tekton.dev", "pipeline_v1.json"), ("tekton.dev", "task_v1.json"), + ("apiextensions.k8s.io", "customresourcedefinition_v1.json"), ) for group, filename in expected: assert (dv.ROOT / "scripts/ci/schemas" / group / filename).is_file() diff --git a/tests/scripts/test_kata_worker_check.py b/tests/scripts/test_kata_worker_check.py new file mode 100644 index 000000000..1462e0cda --- /dev/null +++ b/tests/scripts/test_kata_worker_check.py @@ -0,0 +1,42 @@ +import importlib.util +import json +from pathlib import Path + +import pytest + + +path = Path(__file__).resolve().parents[2] / "scripts/kata_worker_check.py" +spec = importlib.util.spec_from_file_location("kata_worker_check", path) +check = importlib.util.module_from_spec(spec) +spec.loader.exec_module(check) + + +def node(*roles): + labels = {"kubernetes.io/arch": "amd64"} + labels.update({"node-role.kubernetes.io/" + role: "" for role in roles}) + return {"metadata": {"name": "worker-1", "labels": labels}, "spec": {}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}} + + +@pytest.mark.parametrize("items", [[], [node("worker", "control-plane")], + [node("worker", "master")], [node()]]) +def test_reject_unsafe_target(items): + with pytest.raises(ValueError): + check.workers(items) + + +@pytest.mark.parametrize("facts, passes", [("legacy-BIOS\nKVM", True), ("UEFI\nmissing-KVM", False), + ("", False), ("UEFI\nKVM", True)]) +def test_probe(monkeypatch, facts, passes): + monkeypatch.setenv("KATA_NODE_SELECTOR", "role=dedicated") + monkeypatch.setenv("KATA_API_SERVER", "https://cluster") + pod = {"metadata": {"name": "mcd"}, "status": {"containerStatuses": [ + {"name": "machine-config-daemon", "ready": True}]}} + replies = iter(["https://cluster", json.dumps({"items": [node("worker")]}), + json.dumps({"items": [pod]}), facts]) + monkeypatch.setattr(check, "oc", lambda *args: next(replies)) + if passes: + check.main() + else: + with pytest.raises(ValueError, match="must expose /dev/kvm"): + check.main() diff --git a/tests/scripts/test_keycloak_admin_scope.py b/tests/scripts/test_keycloak_admin_scope.py new file mode 100644 index 000000000..590ba6778 --- /dev/null +++ b/tests/scripts/test_keycloak_admin_scope.py @@ -0,0 +1,75 @@ +"""Admin API scope migration is shared, additive and fail-closed on lookup errors.""" +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = ROOT / "charts/openshell-keycloak/files/saw_keycloak_admin.py" +spec = importlib.util.spec_from_file_location("keycloak_admin_scope", SOURCE) +helper = importlib.util.module_from_spec(spec) +spec.loader.exec_module(helper) +SCOPE = {"id": "scope-admin", "name": "kc-admin-api", "protocol": "openid-connect"} + + +def test_rhdh_packages_the_same_helper(): + assert SOURCE.read_bytes() == (ROOT / "charts/openshell-rhdh/files/saw_keycloak_admin.py").read_bytes() + + +def test_scope_migration_is_additive_and_idempotent(): + defaults = [{"id": "profile-id", "name": "profile"}, {"id": "email-id", "name": "email"}] + writes = [] + + def call(method, path, body=None): + if method == "GET": + return [SCOPE] if path == "/client-scopes" else defaults + writes.append((method, path, body)) + defaults.append(SCOPE) + + assert helper.ensure_admin_api_scope(call, "reader-client") is True + assert helper.ensure_admin_api_scope(call, "reader-client") is True + assert writes == [("PUT", "/clients/reader-client/default-client-scopes/scope-admin", None)] + assert [scope["name"] for scope in defaults] == ["profile", "email", "kc-admin-api"] + + +def test_older_server_without_builtin_scope_is_unchanged(): + calls = [] + + def call(method, path, body=None): + calls.append((method, path)) + return [{"id": "profile-id", "name": "profile", "protocol": "openid-connect"}] + + assert helper.ensure_admin_api_scope(call, "reader-client") is False + assert calls == [("GET", "/client-scopes")] + + +@pytest.mark.parametrize("fail_path", ["/client-scopes", "/clients/reader-client/default-client-scopes"]) +def test_lookup_failure_never_becomes_a_compatibility_skip(fail_path): + def call(method, path, body=None): + if path == fail_path: + raise PermissionError("lookup denied") + assert method == "GET" + return [SCOPE] + + with pytest.raises(PermissionError, match="lookup denied"): + helper.ensure_admin_api_scope(call, "reader-client") + + +@pytest.mark.parametrize("scopes", [None, {}, [SCOPE, SCOPE], [{**SCOPE, "protocol": "saml"}], + [{**SCOPE, "id": ""}]]) +def test_invalid_or_ambiguous_scope_fails_without_writes(scopes): + def call(method, path, body=None): + assert (method, path) == ("GET", "/client-scopes") + return scopes + + with pytest.raises(RuntimeError, match="invalid Keycloak"): + helper.ensure_admin_api_scope(call, "reader-client") + + +def test_relay_credentials_are_not_granted_admin_api_scope(): + # Dynamic registration uses a separate registrar. Only the approved + # relay scope and basic identity belong in the workspace client. + relay = (ROOT / "charts/saw-workspace-relay/files/identity.py").read_text() + assert 'defaultClientScopes=["basic", "saw-workspace-relay"]' in relay + assert '"grant_type":"password"' not in relay + assert "ensure_admin_api_scope" not in relay diff --git a/tests/scripts/test_keycloak_setup.py b/tests/scripts/test_keycloak_setup.py index 482b67915..9d181fa57 100644 --- a/tests/scripts/test_keycloak_setup.py +++ b/tests/scripts/test_keycloak_setup.py @@ -150,6 +150,9 @@ def test_no_keycloak_deploys_one(kc): (helm,) = kc.calls("helm") assert "keycloak.existing" not in " ".join(helm) and "keycloak.realm=openshell" in helm log = kc.log() + wait = ["oc", "wait", "-n", "keycloak", "--for=condition=Done", + "keycloakrealmimport/openshell-keycloak-realm", "--timeout=5m"] + assert log.index(wait) > log.index(["helm", *helm]) assert not [c for c in log if c[:4] == ["oc", "create", "secret", "generic"]] diff --git a/tests/scripts/test_make_interface.py b/tests/scripts/test_make_interface.py index 468e128dc..358a853fe 100644 --- a/tests/scripts/test_make_interface.py +++ b/tests/scripts/test_make_interface.py @@ -23,6 +23,30 @@ def executable(path, body): path.chmod(0o755) +@pytest.mark.parametrize("target,action", [("saw-create", "apply"), ("saw-preflight", "preflight"), + ("saw-status", "status"), ("saw-delete", "delete")]) +def test_container_make_dispatch_never_calls_vm_operations(tmp_path, target, action): + scripts = tmp_path / "scripts" + scripts.mkdir() + (scripts / "check-saw-name.sh").symlink_to(ROOT / "scripts/check-saw-name.sh") + marker = tmp_path / "called" + (scripts / "container-workspace.py").write_text( + 'import os,sys\nfrom pathlib import Path\nPath(os.environ["MARKER"]).write_text(sys.argv[1])\n') + for filename in ("sandbox-pre-delete.sh", "openshell-saw-create.sh", "saw-delete.sh"): + executable(scripts / filename, "exit 99\n") + result = run_make(target, "OPENSHELL_SAW_NAME=alice", "DEPLOYMENT_MODE=container", + "PYTHON=python3", f"SCRIPTS_DIR={scripts}", env=dict(os.environ, MARKER=str(marker))) + assert result.returncode == 0, result.stderr + assert marker.read_text() == action + + +@pytest.mark.parametrize("target", ["saw-logs", "saw-vm-ssh", "saw-restart", "saw-restart-pending"]) +def test_vm_specific_commands_fail_before_acting_on_container_workspace(target): + result = run_make(target, "OPENSHELL_SAW_NAME=alice", "DEPLOYMENT_MODE=container") + assert result.returncode != 0 + assert "This command operates on VMs" in result.stderr + + def test_help_lists_public_targets_once_without_make_warnings(): result = run_make("help") assert result.returncode == 0, result.stderr @@ -166,7 +190,7 @@ def test_lint_checks_each_shell_file_and_uses_ci_python(tmp_path): bindir = tmp_path / "bin" bindir.mkdir() marker = tmp_path / "lint-calls" - executable(bindir / "helm", 'exit 0\n') + executable(bindir / "helm", 'printf "helm %s\\n" "$*" >> "$MARKER"\n') executable(bindir / "python", 'printf "python %s\\n" "$*" >> "$MARKER"\n') executable(bindir / "uv", 'printf "unexpected uv\\n" >&2\nexit 19\n') bash_stub = bindir / "bash" @@ -182,7 +206,7 @@ def test_lint_checks_each_shell_file_and_uses_ci_python(tmp_path): ("lint", True, True), ("lint-static", True, False), ("lint-interface", False, True), ): marker.write_text("") - result = run_make(target, env=env) + result = run_make(target, "PYTHON=python", env=env) assert result.returncode == 0, result.stderr calls = marker.read_text().splitlines() shell_files = [call.removeprefix("bash ") for call in calls if call.startswith("bash ")] @@ -192,6 +216,40 @@ def test_lint_checks_each_shell_file_and_uses_ci_python(tmp_path): assert ("scripts/test-deployment.sh" in shell_files) == static assert ("python scripts/lint-rendered-shell.py" in calls) == static assert sum(call.startswith("python -m pytest") for call in calls) == int(interface) + helm_calls = [call for call in calls if call.startswith("helm ")] + if static: + # Helm lint deliberately softens template fail() guards and cannot + # accept --api-versions. Real template renders must therefore check + # enabled runtime paths with synthetic discovery as well. + gateway = next(call for call in helm_calls if call.startswith( + "helm template saw-lint charts/openshell-saw-kubernetes")) + assert "-f tests/fixtures/saw-kubernetes-values.yaml" in gateway + assert "--api-versions agents.x-k8s.io/v1beta1/Sandbox" in gateway + assert "--api-versions policy.networking.k8s.io/v1alpha1/AdminNetworkPolicy" in gateway + policy = next(call for call in helm_calls if call.startswith( + "helm template runtime-lint charts/saw-runtime-policies")) + assert "/container-values.yaml" in policy + assert "--api-versions admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" in policy + assert "helm lint charts/agent-sandbox" in helm_calls + assert "helm lint charts/saw-runtime-policies" in helm_calls + assert "helm lint charts/saw-kubernetes-identity" in helm_calls + identity = next(call for call in helm_calls if call.startswith( + "helm template identity-lint charts/saw-kubernetes-identity")) + assert "--namespace saw-lint-trusted" in identity + assert "-f tests/fixtures/saw-kubernetes-identity-values.yaml" in identity + assert "sandboxName" not in identity + assert "helm template agent-sandbox charts/agent-sandbox --namespace agent-sandbox-system --set enabled=true" in helm_calls + + +def test_default_python_suite_includes_offline_kubernetes_runtime(): + # A suite launched by make may inherit its explicit TEST_PATHS through both + # the environment and recursive MAKEFLAGS. Inspect the fresh default here. + env = {key: value for key, value in os.environ.items() + if key not in {"MAKEFLAGS", "MFLAGS", "MAKEOVERRIDES", "TEST_PATHS", "PYTEST_ARGS"}} + result = run_make("--dry-run", "test-installer", "PYTHON=python3", env=env) + assert result.returncode == 0, result.stderr + assert "tests/kubernetes_runtime" in result.stdout + assert "test-stock-protobuf.py" not in result.stdout # optional explicit SDK environment def test_saw_create_preserves_special_characters_in_environment(tmp_path): @@ -536,7 +594,9 @@ def test_platform_builds_report_independently_and_merge_needs_both(name): matrix = build["strategy"]["matrix"] platforms = matrix.get("platform") or [e["platform"] for e in matrix["include"]] assert set(platforms) == {"linux/amd64", "linux/arm64"} - assert merge["needs"] == "build" + publisher = "publish" if "publish" in workflow["jobs"] else "build" + assert merge["needs"] == publisher + assert workflow["jobs"][publisher]["strategy"] == build["strategy"] assert text.count(".github/actions/merge-image/**") == 2 diff --git a/tests/scripts/test_owner_resolver.py b/tests/scripts/test_owner_resolver.py index 300de9168..4a770a083 100644 --- a/tests/scripts/test_owner_resolver.py +++ b/tests/scripts/test_owner_resolver.py @@ -31,7 +31,7 @@ def bootstrap_api(monkeypatch, tmp_path): "publicClient": False, "clientAuthenticatorType": "client-secret", "serviceAccountsEnabled": True, "standardFlowEnabled": False, "directAccessGrantsEnabled": False, "implicitFlowEnabled": False}, - "scopes": [{"id": "admin-scope", "name": "kc-admin-api"}], + "scopes": [{"id": "admin-scope", "name": "kc-admin-api", "protocol": "openid-connect"}], "default_scopes": [], "roles": [{"name": "view-users"}], "calls": [], diff --git a/tests/scripts/test_pattern_cleanup.py b/tests/scripts/test_pattern_cleanup.py index 3c6b5db54..7f41908f0 100644 --- a/tests/scripts/test_pattern_cleanup.py +++ b/tests/scripts/test_pattern_cleanup.py @@ -6,6 +6,8 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] +HCO_SPEC = {"versions": [{"name": "v1", "served": True, "storage": False}, + {"name": "v1beta1", "served": True, "storage": True}]} def fake_oc(tmp_path, namespaces, resources, objects): @@ -16,7 +18,12 @@ def fake_oc(tmp_path, namespaces, resources, objects): (fixtures / "namespaces.json").write_text(json.dumps({"items": namespaces})) (fixtures / "resources.txt").write_text("\n".join(resources) + "\n") for name, items in objects.items(): + if name == "hyperconvergeds.hco.kubevirt.io": + # Only the storage endpoint works after the conversion webhook disappears. + name = "hyperconvergeds.v1beta1.hco.kubevirt.io" (fixtures / f"{name}.json").write_text(json.dumps({"items": items})) + if "hyperconvergeds.hco.kubevirt.io" in objects: + (fixtures / "crd.json").write_text(json.dumps({"spec": HCO_SPEC})) oc = bindir / "oc" oc.write_text('''#!/usr/bin/env bash set -euo pipefail @@ -91,6 +98,7 @@ def test_operator_cleanup_keeps_unowned_resources(tmp_path): assert result.returncode == 0, result.stderr calls = log.read_text().splitlines() assert len(calls) == len(kinds) + assert any(call.startswith("delete hyperconvergeds.v1beta1.hco.kubevirt.io owned ") for call in calls) assert all(" owned -n openshift-cnv " in call for call in calls) assert all("--all" not in call and " other " not in call for call in calls) @@ -99,7 +107,7 @@ def test_operator_cleanup_removes_orphaned_hco_crd(tmp_path): namespace = {"metadata": {"name": "openshift-cnv", "annotations": { "argocd.argoproj.io/tracking-id": "secure-agent-workspace-prod:/Namespace:patterns-operator/openshift-cnv"}}} - crd = {"metadata": {"annotations": { + crd = {"spec": HCO_SPEC, "metadata": {"annotations": { "openshell.pattern/cleanup-on-uninstall": "secure-agent-workspace-prod"}, "labels": { "olm.managed": "true", @@ -115,11 +123,11 @@ def test_operator_cleanup_removes_orphaned_hco_crd(tmp_path): env=env, capture_output=True, text=True) assert result.returncode == 0, result.stderr calls = log.read_text().splitlines() - assert "wait --for=delete namespace/openshift-cnv --timeout=180s" in calls + assert "wait --for=delete namespace/openshift-cnv --timeout=1h" in calls assert "delete crd hyperconvergeds.hco.kubevirt.io --wait=true" in calls -def test_operator_cleanup_warns_when_cnv_namespace_remains(tmp_path): +def test_operator_cleanup_fails_without_removing_crd_when_cnv_namespace_remains(tmp_path): namespace = {"metadata": {"name": "openshift-cnv", "annotations": { "argocd.argoproj.io/tracking-id": "secure-agent-workspace-prod:/Namespace:patterns-operator/openshift-cnv"}}} @@ -127,8 +135,8 @@ def test_operator_cleanup_warns_when_cnv_namespace_remains(tmp_path): env["OC_WAIT_FAIL"] = "yes" result = subprocess.run(["bash", str(ROOT / "scripts/pattern-operator-cleanup.sh")], env=env, capture_output=True, text=True) - assert result.returncode == 0, result.stderr - assert "still exists after 180 s" in result.stderr + assert result.returncode != 0 + assert "CNV teardown incomplete" in result.stderr assert "delete crd" not in log.read_text() @@ -136,7 +144,7 @@ def test_operator_cleanup_preserves_crd_with_hco(tmp_path): namespace = {"metadata": {"name": "openshift-cnv", "annotations": { "argocd.argoproj.io/tracking-id": "secure-agent-workspace-prod:/Namespace:patterns-operator/openshift-cnv"}}} - crd = {"metadata": {"annotations": { + crd = {"spec": HCO_SPEC, "metadata": {"annotations": { "openshell.pattern/cleanup-on-uninstall": "secure-agent-workspace-prod"}, "labels": { "olm.managed": "true", @@ -155,7 +163,7 @@ def test_operator_cleanup_preserves_crd_with_hco(tmp_path): def test_operator_cleanup_removes_marked_crd_after_namespace_is_gone(tmp_path): - crd = {"metadata": {"annotations": { + crd = {"spec": HCO_SPEC, "metadata": {"annotations": { "openshell.pattern/cleanup-on-uninstall": "secure-agent-workspace-prod"}, "labels": { "olm.managed": "true", @@ -175,7 +183,7 @@ def test_operator_cleanup_removes_marked_crd_after_namespace_is_gone(tmp_path): def test_operator_cleanup_keeps_unmarked_crd_after_namespace_is_gone(tmp_path): - crd = {"metadata": {"labels": { + crd = {"spec": HCO_SPEC, "metadata": {"labels": { "olm.managed": "true", "operators.coreos.com/kubevirt-hyperconverged.openshift-cnv": ""}}} env, log = fake_oc(tmp_path, [], [], {}) @@ -190,7 +198,7 @@ def test_cnv_mark_preserves_ownership_before_namespace_removal(tmp_path): namespace = {"metadata": {"annotations": { "argocd.argoproj.io/tracking-id": "secure-agent-workspace-prod:/Namespace:patterns-operator/openshift-cnv"}}} - crd = {"metadata": {"labels": { + crd = {"spec": HCO_SPEC, "metadata": {"labels": { "olm.managed": "true", "operators.coreos.com/kubevirt-hyperconverged.openshift-cnv": ""}}} env, log = fake_oc(tmp_path, [], [], {}) diff --git a/tests/scripts/test_pattern_drain.py b/tests/scripts/test_pattern_drain.py new file mode 100644 index 000000000..cd8866816 --- /dev/null +++ b/tests/scripts/test_pattern_drain.py @@ -0,0 +1,114 @@ +"""Regression coverage for the destructive part of the documented uninstall.""" +import copy +import importlib.util +import subprocess +from pathlib import Path + +import pytest + + +@pytest.fixture +def drain(): + path = Path(__file__).resolve().parents[2] / "scripts/pattern-drain-applications.py" + spec = importlib.util.spec_from_file_location("pattern_drain", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def leaf(): + return {"metadata": {"uid": "old-leaf", "resourceVersion": "11", "name": "openshell-rhdh", + "namespace": "vp-gitops", "finalizers": ["resources-finalizer.argocd.argoproj.io/foreground"], + "labels": {"validatedpatterns.io/pattern": "secure-agent-workspace"}, + "annotations": {"argocd.argoproj.io/tracking-id": + "secure-agent-workspace-prod:argoproj.io/Application:vp-gitops/openshell-rhdh"}}, + "spec": {"destination": {"name": "in-cluster"}, "source": {"path": "charts/openshell-rhdh"}, "syncPolicy": {"automated": {}}}, + "operation": {"sync": {"revision": "test"}}, "status": {"operationState": {"phase": "Running"}}} + + +@pytest.mark.parametrize("change", [ + lambda m: m.update(uid="replacement"), + lambda m: m["labels"].clear(), + lambda m: m["annotations"].update({"argocd.argoproj.io/tracking-id": "other:argoproj.io/Application:vp-gitops/openshell-rhdh"}), + lambda m: m.update(namespace="foreign"), + lambda m: m["finalizers"].clear(), +]) +def test_foreign_or_replaced_leaf_fails_closed(drain, change): + item = leaf() + change(item["metadata"]) + with pytest.raises(RuntimeError, match="unverified or replaced"): + drain.validate_child(item, "secure-agent-workspace-prod", "openshell-rhdh", "old-leaf") + + +def test_only_a_local_foreground_leaf_can_be_drained(drain): + item = leaf() + drain.validate_child(item, "secure-agent-workspace-prod", "openshell-rhdh", "old-leaf") + item["spec"]["destination"] = {"name": "other-cluster"} + with pytest.raises(RuntimeError, match="unverified or replaced"): + drain.validate_child(item, "secure-agent-workspace-prod", "openshell-rhdh", "old-leaf") + + +def test_prune_requires_parent_to_release_the_exact_namespaced_application(drain): + resource = {"group": "argoproj.io", "kind": "Application", "namespace": "vp-gitops", "name": "openshell-rhdh"} + parent = {"status": {"resources": [resource]}} + assert not drain.released(parent, "openshell-rhdh") + resource["requiresPruning"] = True + assert drain.released(parent, "openshell-rhdh") + resource["namespace"] = "foreign" + assert not drain.released(parent, "openshell-rhdh") + + +def test_termination_precedes_preconditioned_foreground_delete(drain, monkeypatch): + item = leaf() + writes = [] + monkeypatch.setattr(drain, "oc", lambda *args, body: writes.append((args, copy.deepcopy(body)))) + drain.drain_child(item) + assert writes[-1][1][-1] == {"op": "remove", "path": "/spec/syncPolicy/automated"} + item["spec"]["syncPolicy"].clear() + drain.drain_child(item) + patch = writes[-1][1] + assert patch[:2] == [{"op": "test", "path": "/metadata/uid", "value": "old-leaf"}, + {"op": "test", "path": "/metadata/resourceVersion", "value": "11"}] + assert patch[-1] == {"op": "replace", "path": "/status/operationState/phase", "value": "Terminating"} + item["status"]["operationState"]["phase"] = "Terminating" + drain.drain_child(item) + assert len(writes) == 2 + item.pop("operation") + drain.drain_child(item) + args, body = writes[-1] + assert args[:2] == ("delete", "--raw") and args[2].endswith("/namespaces/vp-gitops/applications/openshell-rhdh") + assert body["preconditions"] == {"uid": "old-leaf", "resourceVersion": "11"} + assert body["propagationPolicy"] == "Foreground" + + +def test_disable_is_guarded_by_pattern_uid_and_entire_spec(drain, monkeypatch): + pattern = {"metadata": {"uid": "pattern-uid", "resourceVersion": "3", + "namespace": "patterns-operator", "name": drain.PATTERN}, + "spec": {"clusterGroupName": "prod", "extraParameters": []}} + writes = [] + monkeypatch.setattr(drain, "oc", lambda *args, body: writes.append(body)) + drain.disable_consumers(pattern, ["saw-users"]) + assert writes[0][:2] == [ + {"op": "test", "path": "/metadata/uid", "value": "pattern-uid"}, + {"op": "test", "path": "/spec", "value": pattern["spec"]}] + assert writes[0][2]["path"] == "/spec/extraParameters/-" + + +def test_disable_is_idempotent_and_preserves_other_site_overrides(drain, monkeypatch): + key = "clusterGroup.applications.openshell-rhdh.disabled" + pattern = {"spec": {"extraParameters": [{"name": "unrelated", "value": "keep"}, + {"name": key, "value": "false"}]}} + patches = [] + monkeypatch.setattr(drain, "patch", lambda kind, obj, edits: patches.append(edits)) + drain.disable_consumers(pattern, ["openshell-rhdh"]) + assert patches == [[{"op": "replace", "path": "/spec/extraParameters/1/value", "value": "true"}]] + pattern["spec"]["extraParameters"][1]["value"] = "true" + drain.disable_consumers(pattern, ["openshell-rhdh"]) + assert len(patches) == 1 + + + +def test_documented_target_drains_writers_before_vm_cleanup(): + root = Path(__file__).resolve().parents[2] + commands = subprocess.check_output(["make", "-n", "pre-uninstall"], cwd=root, text=True) + assert commands.index("pattern-drain-applications.py") < commands.index("pattern-pre-uninstall.sh") diff --git a/tests/scripts/test_profile_catalog_automation.py b/tests/scripts/test_profile_catalog_automation.py index 8d0c308c7..a6943b46b 100644 --- a/tests/scripts/test_profile_catalog_automation.py +++ b/tests/scripts/test_profile_catalog_automation.py @@ -18,6 +18,8 @@ GENERATOR = ROOT / "scripts" / "saw-profile-catalog.py" WORKFLOW = ROOT / ".github" / "workflows" / "profile-catalog.yml" HOOK = ROOT / ".githooks" / "pre-commit" +UI_SOURCE = "charts/openshell-saw/files/installer/saw_ui_limit.py" +UI_MIRROR = "charts/saw-workspace-relay/files/saw_ui_limit.py" def outputs(): @@ -63,12 +65,17 @@ def test_attached_model_fields_follow_enabled_sandbox_providers(scratch, enabled def test_list_names_every_generated_file(): listed = outputs() - # The six catalog/form files, plus the installer's mirror of every - # shipped governance profile. + # Catalogs, owner/admin forms, shared broker inputs and the governance mirrors. mirrored = [p for p in listed if p.startswith("charts/openshell-saw/files/provider-profiles/")] shipped = sorted(p.name for p in (ROOT / "charts/governance-policy/profiles").glob("*.yaml")) # ... plus the mirror directory itself (so a removed mirror is staged too). - assert len(listed) == 6 + len(shipped) + 1 + assert "charts/saw-users/templates/_governed-proxy-policy.tpl" in listed + assert "charts/saw-users/files/governed-profiles.json" in listed + assert "charts/openshell-rhdh/files/set-workspace-state.yaml" in listed + assert "charts/openshell-rhdh/files/workspace.py" in listed + assert "charts/openshell-rhdh/files/saw_keycloak_admin.py" in listed + assert UI_MIRROR in listed + assert len(listed) == 12 + len(shipped) + 1 assert sorted(p.rsplit("/", 1)[1] for p in mirrored) == shipped assert "charts/openshell-saw/files/provider-profiles" in listed assert "charts/openshell-rhdh/files/create-workspace.yaml" in listed @@ -81,7 +88,7 @@ def scratch(tmp_path): """A git repo with the generator, the profiles and their outputs, and a PATH whose python/python3 are this interpreter (it has PyYAML).""" repo = tmp_path / "repo" - for rel in ["scripts/saw-profile-catalog.py", ".githooks/pre-commit", *outputs()]: + for rel in ["scripts/saw-profile-catalog.py", ".githooks/pre-commit", "charts/openshell-saw/templates/_governed-proxy-policy.tpl", "charts/saw-users/files/workspace.py", "charts/openshell-keycloak/files/saw_keycloak_admin.py", UI_SOURCE, *outputs()]: if (ROOT / rel).is_dir(): # the mirror directory is listed as an owned path continue (repo / rel).parent.mkdir(parents=True, exist_ok=True) @@ -233,6 +240,26 @@ def test_the_workflow_runs_on_profile_changes_without_pull_request_target(): assert "pull_request_target" not in on assert "charts/saw-bom/profiles/**" in on["pull_request"]["paths"] assert doc["permissions"] == {"contents": "write"} + for event in ("pull_request", "push"): + assert UI_SOURCE in on[event]["paths"] + assert UI_MIRROR in on[event]["paths"] + + +@pytest.mark.parametrize("changed", [UI_SOURCE, UI_MIRROR]) +def test_transport_source_or_mirror_only_change_runs_generation(scratch, changed): + repo, env, git = scratch + target = repo / changed + target.write_text(target.read_text() + "\n# transport fixture change\n") + git("add", changed) + result = subprocess.run(["bash", ".githooks/pre-commit"], cwd=repo, env=env, + capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + assert (repo / UI_MIRROR).read_bytes() == (repo / UI_SOURCE).read_bytes() + assert git("show", ":" + UI_MIRROR).stdout == (repo / UI_SOURCE).read_text() + if changed == UI_SOURCE: + assert UI_MIRROR in git("diff", "--cached", "--name-only").stdout + else: + assert git("diff", "--cached", "--name-only").stdout == "" def test_the_hook_is_executable(): diff --git a/tests/scripts/test_provider_inference_smoke.py b/tests/scripts/test_provider_inference_smoke.py new file mode 100644 index 000000000..23356d153 --- /dev/null +++ b/tests/scripts/test_provider_inference_smoke.py @@ -0,0 +1,43 @@ +"""Exercise the live probe's wire contracts with local, credential-free fixtures.""" +import os +from pathlib import Path +import shutil +import subprocess + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.mark.skipif(not shutil.which("node"), reason="node required") +@pytest.mark.parametrize("kind", ["openai", "anthropic", "nvidia"]) +@pytest.mark.parametrize("failure", ["none", "http", "wrong-reply", "network"]) +def test_live_probe_wire_contract_and_sanitized_output(kind, failure): + prelude = """ +const kind = process.argv[1]; +globalThis.fetch = async (url, options) => { + const body = JSON.parse(options.body); + const host = {openai:'api.openai.com',anthropic:'api.anthropic.com',nvidia:'integrate.api.nvidia.com'}[kind]; + if (new URL(url).hostname !== host || options.redirect !== 'error') throw Error('invalid destination'); + const key = kind === 'anthropic' ? options.headers['x-api-key'] : options.headers.authorization; + if (!key?.includes('fixture-placeholder')) throw Error('missing placeholder'); + if (kind === 'anthropic' && options.headers['anthropic-version'] !== '2023-06-01') throw Error('missing version'); + if (body[kind === 'openai' ? 'max_completion_tokens' : 'max_tokens'] !== 128) throw Error('missing bound'); + if (process.env.FIXTURE_FAILURE === 'network') throw Error('DO-NOT-PRINT-PRIVATE-ERROR'); + const reply = process.env.FIXTURE_FAILURE === 'wrong-reply' ? 'unexpected' : body.messages[0].content.split(': ').pop(); + return {ok:process.env.FIXTURE_FAILURE !== 'http',status:process.env.FIXTURE_FAILURE === 'http' ? 401 : 200, + json:async()=>kind === 'anthropic' ? {content:[{type:'text',text:reply}]} : {choices:[{message:{content:reply}}]}}; +}; +""" + # Keep production declaration names independent from fixture variables. + prelude = prelude.replace("const kind =", "const fixtureKind =").replace("[kind]", "[fixtureKind]").replace("kind ===", "fixtureKind ===") + environment = {**os.environ, "FIXTURE_FAILURE":failure, + "OPENAI_API_KEY":"fixture-placeholder", "ANTHROPIC_API_KEY":"fixture-placeholder", + "NVIDIA_API_KEY":"fixture-placeholder"} + result = subprocess.run(["node", "--input-type=module", "-e", prelude + (ROOT/"tests/provider-inference-smoke.mjs").read_text(), + kind, "fixture-model"], env=environment, capture_output=True, text=True) + assert (result.returncode == 0) == (failure == "none"), result.stderr + assert "fixture-placeholder" not in result.stdout + result.stderr + assert "DO-NOT-PRINT-PRIVATE-ERROR" not in result.stdout + result.stderr + if failure == "none": + assert '"nonceMatched":true' in result.stdout diff --git a/tests/scripts/test_redirect_registrar.py b/tests/scripts/test_redirect_registrar.py index b00ae8f86..d7fb76ace 100644 --- a/tests/scripts/test_redirect_registrar.py +++ b/tests/scripts/test_redirect_registrar.py @@ -488,7 +488,7 @@ def test_bootstrap_admin_scope(world, existing_client, scope_state): if existing_client: reg.bootstrap() cid = "c-2" - scope = {"id": "admin-scope", "name": "kc-admin-api"} + scope = {"id": "admin-scope", "name": "kc-admin-api", "protocol": "openid-connect"} other = {"id": "profile-scope", "name": "profile"} kc.scopes = [other] if scope_state == "absent" else [other, scope] kc.default_scopes[cid] = [other] + ([scope] if scope_state == "assigned" else []) diff --git a/tests/scripts/test_rhdh_bootstrap_check.py b/tests/scripts/test_rhdh_bootstrap_check.py new file mode 100644 index 000000000..a7007c1c3 --- /dev/null +++ b/tests/scripts/test_rhdh_bootstrap_check.py @@ -0,0 +1,31 @@ +import os +import subprocess +from pathlib import Path + +import pytest + + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts/check-rhdh-bootstrap.sh" + + +@pytest.mark.parametrize(("available", "replicas"), [(True, 1), (False, 1), (True, 0)]) +def test_check_requires_available_portal_even_when_argo_is_healthy(tmp_path, available, replicas): + calls = tmp_path / "calls" + oc = tmp_path / "oc" + oc.write_text("""#!/bin/sh +echo "$*" >> "$CALLS" +case "$*" in + 'get applications.argoproj.io/'*) echo '{"status":{"sync":{"status":"Synced"},"health":{"status":"Healthy"}}}' ;; + 'get deployment/'*) printf '{"metadata":{"generation":2},"spec":{"replicas":%s},"status":{"observedGeneration":2,"updatedReplicas":%s,"availableReplicas":%s}}' "$REPLICAS" "$REPLICAS" "$REPLICAS" ;; + 'wait --for=condition=Available deployment/'*) exit "$AVAILABLE_RC" ;; +esac +""") + oc.chmod(0o755) + env = dict(os.environ, CALLS=str(calls), AVAILABLE_RC="0" if available else "1", + REPLICAS=str(replicas), RHDH_NAME="demo", RHDH_NAMESPACE="test-rhdh", + PATH=f"{tmp_path}:{os.environ['PATH']}") + result = subprocess.run([str(SCRIPT), "after"], env=env, capture_output=True, text=True) + log = calls.read_text().splitlines() + assert "wait --for=condition=Available deployment/backstage-demo -n test-rhdh --timeout=30m" in log + assert ("rollout status deployment/backstage-demo -n test-rhdh --timeout=30m" in log) == available + assert (result.returncode == 0) == (available and replicas > 0) diff --git a/tests/scripts/test_saw_preflight.py b/tests/scripts/test_saw_preflight.py new file mode 100644 index 000000000..c8a288b3f --- /dev/null +++ b/tests/scripts/test_saw_preflight.py @@ -0,0 +1,383 @@ +import importlib.util +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = ROOT / "charts/saw-portal-container/files/preflight.py" +spec = importlib.util.spec_from_file_location("saw_preflight", MODULE_PATH) +preflight = importlib.util.module_from_spec(spec) +spec.loader.exec_module(preflight) + + +def runtime_class(): + obj = { + "metadata": { + "name": "approved-kata", + "uid": "runtime-uid-1", + "labels": {"openshell.ai/sandbox-attachable": "true"}, + }, + "handler": "kata", + "scheduling": {"nodeSelector": {"workload": "kata"}}, + "overhead": {"podFixed": {"memory": "128Mi"}}, + } + approval = { + "name": obj["metadata"]["name"], + "uid": obj["metadata"]["uid"], + "sha256": preflight.runtime_placement_digest(obj), + } + return obj, approval + + +def test_runtime_approval_passes_for_the_same_uid_handler_and_placement(): + observed, approved = runtime_class() + assert preflight.check_runtime_class(approved, observed) + + +@pytest.mark.parametrize(("change", "reason"), [ + ("missing", "RuntimeClassUnavailable"), + ("name", "RuntimeClassNameMismatch"), + ("uid", "RuntimeClassIdentityMismatch"), + ("handler", "RuntimeClassHandlerMismatch"), + ("placement", "RuntimeClassPlacementChanged"), + ("label", "RuntimeClassNotAttachable"), + ("scheduling", "RuntimeClassSchedulingInvalid"), + ("approval", "RuntimeClassApprovalInvalid"), +]) +def test_runtime_approval_reports_the_failed_boundary(change, reason): + observed, approved = runtime_class() + if change == "missing": + observed = None + elif change == "name": + observed["metadata"]["name"] = "other-kata" + elif change == "uid": + observed["metadata"]["uid"] = "replacement-uid" + elif change == "handler": + observed["handler"] = "runc" + elif change == "placement": + observed["scheduling"]["nodeSelector"]["workload"] = "general" + elif change == "label": + observed["metadata"]["labels"]["openshell.ai/sandbox-attachable"] = "false" + elif change == "scheduling": + observed["scheduling"] = {} + elif change == "approval": + approved["sha256"] = "bad" + with pytest.raises(preflight.PreflightError, match=reason): + preflight.check_runtime_class(approved, observed) + + +def test_runtime_approval_missing_site_binding_has_a_fixed_reason(): + observed, _ = runtime_class() + with pytest.raises(preflight.PreflightError, match="RuntimeClassApprovalMissing"): + preflight.check_runtime_class(None, observed) + + +def test_validation_policy_hash_normalizes_registration_specific_names(): + registration = "0123456789abcdef" + installation = "11111111-1111-4111-8111-111111111111" + slots = ["default/worker", "tools/worker"] + + def spec(reg, install): + relays = [preflight.hashlib.sha256((reg + "/" + slot).encode()).hexdigest()[:16] + for slot in slots] + return { + "controlNamespace": "saw-ctl-" + reg, + "installation": install, + "gatewaySlots": ["alice-" + relay[:8] for relay in relays], + "gateRelays": ["gate-" + relay + "-claims" for relay in relays], + "browserProofs": ["ui-" + relay + "-browser-proof" for relay in relays], + } + + first = preflight.validation_policy_digest( + spec(registration, installation), "alice", registration, installation, slots) + next_registration = "fedcba9876543210" + next_installation = "22222222-2222-4222-8222-222222222222" + second = preflight.validation_policy_digest( + spec(next_registration, next_installation), "alice", next_registration, + next_installation, slots) + assert first == second + assert len(first) == 64 + + +def test_boundary_reference_fails_closed_on_missing_or_changed_hash(): + observed = "a" * 64 + assert preflight.verify_boundary_hash({"validation": observed}, "validation", observed) + with pytest.raises(preflight.PreflightError, match="BoundaryApprovalMissing"): + preflight.verify_boundary_hash({}, "validation", observed) + with pytest.raises(preflight.PreflightError, match="BoundaryApprovalChanged"): + preflight.verify_boundary_hash({"validation": "b" * 64}, "validation", observed) + + +@pytest.mark.parametrize(("mode", "reference"), [ + ("container", {"validation": "a" * 64, "binding": "b" * 64}), + ("kata", {"validation": "a" * 64, "binding": "b" * 64, + "sandboxPlacement": "c" * 64, "sandboxPlacementBinding": "d" * 64}), +]) +def test_boundary_reference_requires_all_hashes_for_the_selected_mode(mode, reference): + assert preflight.verify_boundary_reference(reference, mode) + + +@pytest.mark.parametrize(("mode", "reference", "reason"), [ + ("kata", {"validation": "a" * 64}, "BoundaryApprovalMissing"), + ("kata", {"validation": "a" * 64, "binding": "b" * 64, + "sandboxPlacement": "not-a-hash", "sandboxPlacementBinding": "d" * 64}, + "BoundaryApprovalInvalid"), +]) +def test_boundary_reference_rejects_incomplete_or_malformed_hashes(mode, reference, reason): + with pytest.raises(preflight.PreflightError, match=reason): + preflight.verify_boundary_reference(reference, mode) + + +def observations(): + return {key: True for key in preflight.COMMON_CHECKS + preflight.KATA_CHECKS} + + +def test_all_kata_preflight_facts_must_pass(): + facts = observations() + assert len(preflight.check_observations("kata", facts)) == len(facts) + + +@pytest.mark.parametrize(("fact", "reason"), [ + ("argo", "ArgoAccessDenied"), + ("agentSandbox", "AgentSandboxUnavailable"), + ("storage", "StorageUnavailable"), + ("providerEgress", "ProviderEgressRejected"), + ("registry", "RegistryUnavailable"), + ("osc", "SandboxedContainersUnavailable"), + ("kataConfig", "KataConfigNotReady"), + ("workers", "KataWorkersNotReady"), +]) +def test_preflight_fails_closed_on_missing_prerequisites(fact, reason): + facts = observations() + facts[fact] = False + with pytest.raises(preflight.PreflightError, match=reason): + preflight.check_observations("kata", facts) + + +def test_non_kata_mode_does_not_require_kata_operator_facts(): + facts = {key: True for key in preflight.COMMON_CHECKS} + assert preflight.check_observations("container", facts) == preflight.COMMON_CHECKS + + +def test_known_nontraversable_workload_needs_pinned_replacement(): + source = next(iter(preflight.NON_TRAVERSABLE_WORKLOAD_IMAGES)) + replacement = "registry.example.invalid/openclaw@sha256:" + "a" * 64 + catalog = {"approved": {"workspaces": [{"enabled": True, "sandboxes": [ + {"name": "notebook", "enabled": True, "image": source}, + ]}]}} + with pytest.raises(preflight.PreflightError, match="WorkloadImageNotNamespaceAccessible"): + preflight.verify_profile_image_access({}, "approved", catalog) + site = {"profileImageOverrides": {"approved": {"notebook": replacement}}} + assert preflight.verify_profile_image_access(site, "approved", catalog) + site["profileImageOverrides"]["approved"]["notebook"] = "openclaw:latest" + with pytest.raises(preflight.PreflightError, match="WorkloadImageApprovalInvalid"): + preflight.verify_profile_image_access(site, "approved", catalog) + + +def test_disabled_workload_does_not_require_image_replacement(): + source = next(iter(preflight.NON_TRAVERSABLE_WORKLOAD_IMAGES)) + catalog = {"approved": {"workspaces": [{"enabled": True, "sandboxes": [ + {"name": "notebook", "enabled": False, "image": source}, + ]}]}} + assert preflight.verify_profile_image_access({}, "approved", catalog) + + +def site_config(): + pin = "a" * 40 + image = "registry.example.invalid/worker@sha256:" + "b" * 64 + return { + "providerAllowHosts": ["api.example.invalid"], + "providerPolicy": {"allowHosts": ["api.example.invalid"]}, + "automaticEnrollment": {"revision": pin, "image": image}, + "stack": { + "policy": {"revision": pin}, + "proxy": {"revision": pin, "values": { + "image": image, + "network": {"bootstrapEgress": [{"to": ["api.example.invalid"]}]}, + }}, + "gateway": {"revision": pin, "values": { + "kubernetes": {"storage": {"className": "workspace-rwx"}, "image": image}, + }}, + "audit": {"revision": pin, "values": {"storageClass": "workspace-rwx", "image": image}}, + "identity": {"revision": pin, "values": {"secretStore": "vault-backend", "image": image}}, + }, + } + + +def test_site_configuration_requires_pinned_sources_and_scoped_egress(): + facts = preflight.configuration_facts(site_config()) + assert facts == { + "network": True, + "storage": True, + "secrets": True, + "registry": True, + "providerEgress": True, + "revisions": True, + } + changed = site_config() + changed["providerPolicy"]["allowHosts"] = ["*"] + changed["stack"]["gateway"]["revision"] = "main" + changed["stack"]["audit"]["values"]["image"] = "image:latest" + facts = preflight.configuration_facts(changed) + assert facts["providerEgress"] is False + assert facts["revisions"] is False + assert facts["registry"] is False + missing_component = site_config() + del missing_component["stack"]["audit"] + assert preflight.configuration_facts(missing_component)["revisions"] is False + invalid_network = site_config() + invalid_network["stack"]["proxy"]["values"]["network"]["bootstrapEgress"] = "all" + assert preflight.configuration_facts(invalid_network)["network"] is False + + +def test_documented_provider_hosts_do_not_need_generated_policy(): + site = site_config() + del site["providerPolicy"] + assert preflight.configuration_facts(site)["providerEgress"] is True + for hosts in ([], ["*"], ["api.example.invalid", "*.example.invalid"]): + site["providerAllowHosts"] = hosts + assert preflight.configuration_facts(site)["providerEgress"] is False + site["providerAllowHosts"] = ["api.example.invalid"] + site["providerPolicy"] = {"allowHosts": ["other.example.invalid"]} + assert preflight.configuration_facts(site)["providerEgress"] is False + site["providerPolicy"] = {"allowHosts": [{}]} + assert preflight.configuration_facts(site)["providerEgress"] is False + + +def test_storage_preflight_rejects_missing_class_and_accepts_default_class(): + with pytest.raises(preflight.PreflightError, match="StorageUnavailable"): + preflight.verify_storage_classes(site_config(), []) + default_site = {"stack": {}} + classes = [{"metadata": {"name": "default-rwx", "annotations": { + "storageclass.kubernetes.io/is-default-class": "true", + }}}] + assert preflight.verify_storage_classes(default_site, classes) == () + + +def test_secret_store_must_report_ready(): + site = site_config() + with pytest.raises(preflight.PreflightError, match="SecretServicesUnavailable"): + preflight.verify_secret_store(site, [{"metadata": {"name": "vault-backend"}, + "status": {"conditions": []}}]) + assert preflight.verify_secret_store(site, [{ + "metadata": {"name": "vault-backend"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }]) == "vault-backend" + + +@pytest.mark.parametrize(("health", "ready"), [ + ({"initialized": True, "sealed": False}, True), + ({"initialized": False, "sealed": False}, False), + ({"initialized": True, "sealed": True}, False), +]) +def test_vault_health_rejects_uninitialized_or_sealed_service(health, ready): + assert preflight.vault_health_ready(health) is ready + + +def test_portal_chart_uses_the_shared_preflight_source(): + portal_copy = ROOT / "charts/openshell-rhdh/files/preflight.py" + assert portal_copy.read_bytes() == MODULE_PATH.read_bytes() + projector_copy = ROOT / "charts/saw-users/files/preflight.py" + assert projector_copy.read_bytes() == MODULE_PATH.read_bytes() + + +@pytest.mark.parametrize(("discovery", "available"), [ + ({"groupVersion": "admissionregistration.k8s.io/v1", "resources": [ + {"name": "validatingadmissionpolicies"}, + {"name": "validatingadmissionpolicybindings"}, + ]}, True), + ({"groupVersion": "admissionregistration.k8s.io/v1", "resources": [ + {"name": "validatingadmissionpolicies"}, + ]}, False), + ({"groupVersion": "admissionregistration.k8s.io/v1beta1", "resources": [ + {"name": "validatingadmissionpolicies"}, + {"name": "validatingadmissionpolicybindings"}, + ]}, False), + ({"resources": []}, False), +]) +def test_admission_api_availability_uses_discovery(discovery, available): + assert preflight.api_resources_available( + discovery, "admissionregistration.k8s.io/v1", + ("validatingadmissionpolicies", "validatingadmissionpolicybindings"), + ) is available + + +def test_kata_platform_requires_ready_cluster_kataconfig_workers_and_apis(): + observed, approved = runtime_class() + site = {"kataRuntimeClass": approved} + configs = [{"status": {"conditions": [{"type": "InProgress", "status": "False"}], + "kataNodes": {"nodeCount": 1, "readyNodeCount": 1, + "installed": ["worker-a"]}}}] + nodes = [{ + "metadata": {"name": "worker-a", "labels": { + "kubernetes.io/arch": "amd64", "node-role.kubernetes.io/worker": "", + "workload": "kata", + }}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + "spec": {}, + }] + api_readiness = {name: True for name in ( + "kataconfigs.kataconfiguration.openshift.io", "sandboxes.agents.x-k8s.io", + "externalsecrets.external-secrets.io", "adminnetworkpolicies.policy.networking.k8s.io", + "admissionregistration.k8s.io/v1", + )} + version = {"status": {"conditions": [{"type": "Available", "status": "True"}], + "desired": {"version": "4.22.22"}}} + assert preflight.verify_kata_platform(site, observed, configs, nodes, + api_readiness, version) == ("worker-a",) + + +def test_kata_platform_rejects_no_ready_node_in_approved_placement(): + observed, approved = runtime_class() + nodes = [{ + "metadata": {"name": "worker-a", "labels": { + "kubernetes.io/arch": "amd64", "node-role.kubernetes.io/worker": "", + "workload": "general", + }}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + "spec": {}, + }] + version = {"status": {"conditions": [{"type": "Available", "status": "True"}], + "desired": {"version": "4.22.22"}}} + with pytest.raises(preflight.PreflightError, match="KataWorkersNotReady"): + preflight.verify_kata_platform( + {"kataRuntimeClass": approved}, observed, + [{"status": {"conditions": [{"type": "InProgress", "status": "False"}], + "kataNodes": {"nodeCount": 1, "readyNodeCount": 1, + "installed": ["worker-a"]}}}], + nodes, + {name: True for name in ( + "kataconfigs.kataconfiguration.openshift.io", "sandboxes.agents.x-k8s.io", + "externalsecrets.external-secrets.io", "adminnetworkpolicies.policy.networking.k8s.io", + "admissionregistration.k8s.io/v1", + )}, + version, + ) + + +@pytest.mark.parametrize(("node_count", "ready_count", "in_progress"), [ + (1, 0, "True"), + (2, 1, "False"), + (0, 0, "False"), +]) +def test_kataconfig_must_finish_on_all_selected_workers(node_count, ready_count, in_progress): + config = {"status": { + "conditions": [{"type": "InProgress", "status": in_progress}], + "kataNodes": {"nodeCount": node_count, "readyNodeCount": ready_count}, + }} + assert preflight.kata_config_ready(config) is (node_count > 0 and ready_count == node_count + and in_progress == "False") + + +def test_cluster_version_must_be_available_and_at_least_422(): + available = {"status": {"conditions": [{"type": "Available", "status": "True"}], + "desired": {"version": "4.22.22"}}} + assert preflight.verify_cluster_version(available) == "4.22.22" + with pytest.raises(preflight.PreflightError, match="UnsupportedOpenShiftVersion"): + preflight.verify_cluster_version({"status": { + "conditions": [{"type": "Available", "status": "True"}], + "desired": {"version": "4.21.12"}, + }}) + with pytest.raises(preflight.PreflightError, match="ClusterVersionUnavailable"): + preflight.verify_cluster_version({"status": {"desired": {"version": "4.22.22"}}}) diff --git a/tests/scripts/test_standalone_projector.py b/tests/scripts/test_standalone_projector.py new file mode 100644 index 000000000..ef3a486fc --- /dev/null +++ b/tests/scripts/test_standalone_projector.py @@ -0,0 +1,290 @@ +import copy +import importlib.util +import json +from pathlib import Path +import sys +import threading +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +FILES = ROOT / "charts/saw-users/files" + + +def load(name): + spec = importlib.util.spec_from_file_location(name, FILES / (name+".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def projector(monkeypatch): + monkeypatch.setitem(sys.modules, "workspace", load("workspace")) + monkeypatch.setitem(sys.modules, "preflight", load("preflight")) + return load("projector") + + +def test_gitops_preflight_stops_stale_policy_before_projection(projector): + cfg = config() + owner = cfg["users"][0] + ns = "saw-" + owner["name"] + policy = "saw-" + ns + "-runtime" + + class Reader: + def __call__(self, path): + return {} + + def preflight_resource(self, path): + if path == "/api/v1/namespaces/" + ns: + return {"metadata": {"labels": { + "saw.redhat.com/runtime-owner": owner["installationID"], + "saw.redhat.com/registration": owner["registration"]}}} + if path.endswith("/validatingadmissionpolicies/" + policy): + return {"spec": {"failurePolicy": "Fail"}, + "metadata": {"generation": 1}, + "status": {"observedGeneration": 1, "typeChecking": {}}} + return {} + + with pytest.raises(projector.ProjectionError, match="BoundaryApprovalChanged:validation"): + projector.parameters(cfg, catalog(), Reader()) + + +def test_gitops_rejects_nontraversable_image_before_projection(projector): + cfg = config() + del cfg["containerSites"]["alice"]["profileImageOverrides"]["data-science"]["notebook"] + with pytest.raises(projector.ProjectionError, match="WorkloadImageNotNamespaceAccessible"): + projector.parameters(cfg, catalog(), lambda _: {}) + + +def test_gitops_kata_preflight_checks_live_runtime(projector): + owner = {"name": "alice", "registration": "a" * 16, + "installationID": "installation"} + runtime = {"metadata": {"name": "kata", "uid": "11111111-1111-4111-8111-111111111111", + "labels": {"openshell.ai/sandbox-attachable": "true"}}, + "handler": "kata", "scheduling": {"nodeSelector": {"node-role.kubernetes.io/kata": ""}}, + "overhead": {"podFixed": {"cpu": "250m", "memory": "350Mi"}}} + site = {"automaticEnrollment": {"reference": { + key: "a" * 64 for key in ("validation", "binding", "sandboxPlacement", "sandboxPlacementBinding")}}, + "kataRuntimeClass": {"name": "kata", "uid": runtime["metadata"]["uid"], + "sha256": projector.preflight.runtime_placement_digest(runtime)}, + "targets": {"default/notebook": {}}} + crd = {"status": {"conditions": [{"type": "Established", "status": "True"}]}} + observed = { + "/apis/config.openshift.io/v1/clusterversions/version": {"status": { + "conditions": [{"type": "Available", "status": "True"}], + "desired": {"version": "4.22.22"}}}, + "/apis/node.k8s.io/v1/runtimeclasses/kata": runtime, + "/apis/kataconfiguration.openshift.io/v1/kataconfigs": {"items": [{"status": { + "kataNodes": {"nodeCount": 1, "readyNodeCount": 1, "installed": ["worker-1"]}}}]}, + "/api/v1/nodes": {"items": [{"metadata": {"name": "worker-1", "labels": { + "kubernetes.io/arch": "amd64", "node-role.kubernetes.io/worker": "", + "node-role.kubernetes.io/kata": ""}}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}}]}, + "/apis/admissionregistration.k8s.io/v1": {"groupVersion": "admissionregistration.k8s.io/v1", + "resources": [{"name": "validatingadmissionpolicies"}, + {"name": "validatingadmissionpolicybindings"}]}, + } + for name in ("kataconfigs.kataconfiguration.openshift.io", + "sandboxes.agents.x-k8s.io", "externalsecrets.external-secrets.io", + "adminnetworkpolicies.policy.networking.k8s.io"): + observed["/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + name] = crd + read = lambda path: observed.get(path, {}) + projector.verify_gitops_preflight(owner, site, "kata", read) + runtime["handler"] = "runc" + with pytest.raises(projector.preflight.PreflightError, match="RuntimeClassHandlerMismatch"): + projector.verify_gitops_preflight(owner, site, "kata", read) + + +def test_gitops_preflight_reader_rejects_secret_paths(projector): + reader = object.__new__(projector.Publications) + with pytest.raises(projector.ProjectionError, match="outside the workspace scope"): + reader.preflight_resource("/api/v1/namespaces/saw-alice/secrets/inference") + + +def config(): + site = yaml.safe_load((ROOT/"tests/fixtures/container-site.yaml").read_text())["portal"]["containerSites"]["alice"] + user = dict(name="alice", ownerSubject=site["subject"], registration="a"*16, + installationID="22222222-2222-4222-8222-222222222222", profiles=["data-science"], + values={"deployment": {"mode": "container"}}, pruneOnRemove=True, + intentGeneration=1, vaultPrefix="secret/data/hub/saw-alice/"+"a"*16) + return dict(users=[user], containerSites={"alice": site}, global_={}) + + +def catalog(): + return json.loads((FILES/"profile-catalog.json").read_text())["profiles"] + + +def test_standalone_uses_exact_portal_projection_without_mutating_inputs(projector): + cfg = config() + before = copy.deepcopy(cfg) + params, = projector.parameters(cfg, catalog(), lambda _: {}) + expected = projector.workspace.project_workspace(cfg["users"][0], {"global_": {}, "projection": {"enabled": False}, "ownerResolution": {"enabled": False}}, + cfg["containerSites"]["alice"], catalog(), lambda _: {}) + assert json.loads(params["workspaceValues"]) == expected + assert "values" not in params # reserved by the ApplicationSet plugin protocol + assert cfg == before + assert set(expected["containerSites"]["alice"]["targets"]) == {"default/notebook", "cuda-dev/cuda-sandbox"} + + +def test_standalone_uses_same_selected_revision_for_application_and_children(projector): + cfg = config() + cfg["global"] = {"repoURL": "https://example.com/repo.git", "targetRevision": "main"} + cfg["workspaceRevisions"] = {"alice": "a"*40} + params, = projector.parameters(cfg, catalog(), lambda _: {}) + values = json.loads(params["workspaceValues"]) + assert params["revision"] == values["global"]["targetRevision"] == "a"*40 + assert values["global"]["repoURL"] == cfg["global"]["repoURL"] + cfg["workspaceRevisions"] = {} + params, = projector.parameters(cfg, catalog(), lambda _: {}) + assert params["revision"] == "main" + + +@pytest.mark.parametrize("fault", ["missing-site", "duplicate", "unapproved", "wrong-mode", "broken-publication"]) +def test_projection_failure_never_returns_an_empty_desired_set(projector, fault): + cfg = config() + read = lambda _: {} + if fault == "missing-site": cfg["containerSites"] = {} + if fault == "duplicate": cfg["users"].append(copy.deepcopy(cfg["users"][0])) + if fault == "unapproved": cfg["users"][0]["profiles"] = ["missing"] + if fault == "wrong-mode": cfg["users"][0]["values"]["deployment"]["mode"] = "vm" + if fault == "broken-publication": read = lambda _: {"metadata": {}, "data": {"binding.json": "{}"}} + with pytest.raises(projector.ProjectionError): + projector.parameters(cfg, catalog(), read) + + +class MemoryApi: + def __init__(self): + self.objects = {} + self.created = [] + for ns, group, kind, plural in [("workspaces", "apps/v1", "Deployment", "deployments"), + ("gitops", "argoproj.io/v1alpha1", "ApplicationSet", "applicationsets")]: + self.objects["/api/v1/namespaces/"+ns] = {"metadata": {"uid": ns+"-uid"}} + self.objects[f"/apis/{group}/namespaces/{ns}/{plural}/projector"] = { + "metadata": {"uid": kind+"-uid", "labels": {"saw.redhat.com/projector-owner": "installation"}}} + + def call(self, method, path, body=None): + if method == "GET": return copy.deepcopy(self.objects.get(path)) + assert method == "POST" and path.endswith("/secrets") + name = path+"/"+body["metadata"]["name"] + assert name not in self.objects + self.created.append(name) + self.objects[name] = copy.deepcopy(body) + + +def identity_config(): + return dict(namespace="workspaces", argoNamespace="gitops", name="projector", owner="installation") + + +def test_plugin_identity_is_random_retained_projected_and_garbage_collected_with_owners(): + identity = load("projector_identity") + api = MemoryApi() + identity.provision(identity_config(), api) + first = api.objects["/api/v1/namespaces/workspaces/secrets/projector"] + peer = api.objects["/api/v1/namespaces/gitops/secrets/projector"] + assert first["data"] == peer["data"] + assert first["immutable"] and peer["immutable"] + assert first["metadata"]["ownerReferences"][0]["kind"] == "Deployment" + assert peer["metadata"]["ownerReferences"][0]["kind"] == "ApplicationSet" + identity.provision(identity_config(), api) + assert len(api.created) == 2 + + +@pytest.mark.parametrize("fault", ["owner", "namespace", "token", "mutable"]) +def test_plugin_identity_never_overwrites_foreign_or_changed_material(fault): + identity = load("projector_identity") + api = MemoryApi() + identity.provision(identity_config(), api) + obj = api.objects["/api/v1/namespaces/workspaces/secrets/projector"] + if fault == "owner": obj["metadata"]["ownerReferences"][0]["uid"] = "replacement" + if fault == "namespace": api.objects["/api/v1/namespaces/workspaces"]["metadata"]["uid"] = "replacement" + if fault == "token": obj["data"]["token"] = "invalid" + if fault == "mutable": obj["immutable"] = False + with pytest.raises(identity.IdentityError): identity.provision(identity_config(), api) + assert len(api.created) == 2 + + +def test_http_plugin_authentication_and_error_are_not_a_prune_request(projector, tmp_path): + settings, profiles, token = [tmp_path / name for name in ("config.json", "catalog.json", "token")] + settings.write_text(json.dumps(config())) + profiles.write_text(json.dumps({"profiles": catalog()})) + token.write_text("test-only-plugin-token") + with projector.server(settings, profiles, token, lambda _: {}, ("127.0.0.1", 0)) as server: + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + url = "http://127.0.0.1:"+str(server.server_address[1])+"/api/v1/getparams.execute" + def call(bearer): + req = Request(url, method="POST", data=b"{}", headers={"Authorization": "Bearer "+bearer}) + try: + with urlopen(req, timeout=2) as res: return res.status, json.load(res) + except HTTPError as error: + with error: return error.code, json.load(error) + try: + assert call("wrong")[0] == 403 + code, result = call(token.read_text()) + assert code == 200 and result["output"]["parameters"][0]["name"] == "alice" + assert "workspaceValues" in result["output"]["parameters"][0] + settings.write_text("invalid json") + code, result = call(token.read_text()) + assert code == 503 and "output" not in result + settings.write_text(json.dumps({"users": [], "containerSites": {}})) + assert call(token.read_text()) == (200, {"output": {"parameters": []}}) + finally: + server.shutdown() + worker.join(timeout=3) + + +def test_missing_peer_projection_resumes_with_the_same_identity(): + identity = load("projector_identity") + api = MemoryApi() + identity.provision(identity_config(), api) + first = copy.deepcopy(api.objects["/api/v1/namespaces/workspaces/secrets/projector"]) + del api.objects["/api/v1/namespaces/gitops/secrets/projector"] + identity.provision(identity_config(), api) + assert api.objects["/api/v1/namespaces/workspaces/secrets/projector"] == first + assert api.objects["/api/v1/namespaces/gitops/secrets/projector"]["data"] == first["data"] + + +def test_orphaned_plugin_identity_is_not_silently_replaced(): + identity = load("projector_identity") + api = MemoryApi() + identity.provision(identity_config(), api) + del api.objects["/api/v1/namespaces/workspaces/secrets/projector"] + with pytest.raises(identity.IdentityError, match="orphaned"): + identity.provision(identity_config(), api) + assert len(api.created) == 2 + + +def test_missing_publication_holds_only_an_already_bound_owner(projector): + cfg = config() + bob = copy.deepcopy(cfg["users"][0]) + bob.update(name="bob", registration="b"*16, installationID="33333333-3333-4333-8333-333333333333") + cfg["users"].append(bob) + cfg["containerSites"]["bob"] = copy.deepcopy(cfg["containerSites"]["alice"]) + cfg.setdefault("workspaceRevisions", {})["bob"] = "c"*40 + initial, _ = projector.parameters(cfg, catalog(), lambda _: {}) + last = json.loads(initial["workspaceValues"]) + last["containerSites"]["alice"]["enrollment"] = {"registration": "a"*16} + + class Reader: + def __call__(self, path): + return {} + def previous(self, owner): + return copy.deepcopy(last) if owner["name"] == "alice" else None + + a, b = projector.parameters(cfg, catalog(), Reader()) + assert json.loads(a["workspaceValues"]) == last + assert b["name"] == "bob" and b["revision"] == "c"*40 + assert "enrollment" not in json.loads(b["workspaceValues"])["containerSites"]["bob"] + + +@pytest.mark.parametrize("field",["reference","relay","revision","image","buildNamespace"]) +def test_invalid_enabled_site_reports_owner_and_field(projector, field): + cfg = config() + auto = cfg["containerSites"]["alice"]["automaticEnrollment"] + auto.pop(field) + with pytest.raises(projector.ProjectionError, match="alice.*"+field): + projector.parameters(cfg, catalog(), lambda _: {}) diff --git a/tests/scripts/test_workspace_projection.py b/tests/scripts/test_workspace_projection.py new file mode 100644 index 000000000..5e5dbbe01 --- /dev/null +++ b/tests/scripts/test_workspace_projection.py @@ -0,0 +1,124 @@ +"""The portal adapter and other installation entry points share one projection.""" +import copy +import importlib.util +import json +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("shared_workspace", ROOT / "charts/saw-users/files/workspace.py") +workspace = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(workspace) +CATALOG = json.loads((ROOT / "charts/saw-users/files/profile-catalog.json").read_text())["profiles"] + + +def inputs(): + owner = dict(name="alice", ownerSubject="11111111-1111-4111-8111-111111111111", + installationID="22222222-2222-4222-8222-222222222222", registration="a"*16, + profiles=["data-science"], values={"deployment": {"mode": "container"}}, + lifecycleVersion=1, initialModelInputs={"inference.model": "first"}, + modelInputs={"inference.model": "second"}, desiredState="Running", restartGeneration=2) + site = dict(subject=owner["ownerSubject"], automaticEnrollment={"approvedProfiles": {"data-science": "b"*64}}, + profileImageOverrides={"data-science": {"default": {"notebook": "example/agent@sha256:"+"c"*64}}}) + return owner, site + + +def test_projection_uses_catalog_targets_and_preserves_callers_inputs(): + owner, site = inputs() + before = copy.deepcopy((owner, site)) + result = workspace.project_workspace(owner, {"defaults": {}}, site, CATALOG) + projected = result["containerSites"]["alice"] + assert set(projected["targets"]) == {"default/notebook", "cuda-dev/cuda-sandbox"} + assert projected["modelInputs"] == {"inference.model": "first"} + assert projected["requestedModels"] == {"inference.model": "second"} + assert projected["restartGeneration"] == 2 + assert projected["selectedDigest"] == "b"*64 + assert result["users"][0]["imageOverrides"] == site["profileImageOverrides"]["data-science"] + assert (owner, site) == before + assert "enrollment" not in projected + + +@pytest.mark.parametrize("fault", ["subject", "profile", "identity", "endpoint"]) +def test_projection_rejects_unbound_input(fault): + owner, site = inputs() + if fault == "subject": site["subject"] = "replacement-owner" + if fault == "profile": owner["profiles"] = ["personal-assistant"] + if fault == "identity": del owner["installationID"] + if fault == "endpoint": owner["customInferenceURL"] = "https://unexpected.example/v1" + with pytest.raises(workspace.WorkspaceError): + workspace.project_workspace(owner, {}, site, CATALOG) + + +def test_legacy_vm_remains_vm_and_reserved_modes_never_fall_back(): + result = workspace.project_workspace({"name": "alice"}, {}, {}, CATALOG) + assert result["users"][0]["values"]["deployment"]["mode"] == "vm" + for mode in ("unknown", "confidential"): + with pytest.raises(workspace.WorkspaceError, match="unsupported"): + workspace.project_workspace({"name": "alice", "values": {"deployment": {"mode": mode}}}, {}, {}, CATALOG) + + +def test_packaged_projector_is_exactly_the_canonical_source(): + assert (ROOT / "charts/openshell-rhdh/files/workspace.py").read_bytes() == Path(SPEC.origin).read_bytes() + + +def test_disabling_enrollment_cannot_silently_prune_an_existing_runtime(): + owner, site = inputs() + site["automaticEnrollment"]["enabled"] = False + observed = {"immutable": True, "metadata": {"labels": {"saw.redhat.com/runtime-owner": owner["installationID"]}}, + "data": {"binding.json": "{}"}} + with pytest.raises(workspace.WorkspaceError, match="cannot disable"): + workspace.project_workspace(owner, {}, site, CATALOG, lambda _: observed) + projected = workspace.project_workspace(owner, {}, site, CATALOG, lambda _: {}) + assert "enrollment" not in projected["containerSites"]["alice"] + + +def test_foundation_requires_no_synthetic_approval_hash(): + owner, site = inputs() + site["automaticEnrollment"] = {"enabled": False} + result = workspace.project_workspace(owner, {}, site, CATALOG, lambda _: {}) + assert not result["containerSites"]["alice"]["selectedDigest"] + site["automaticEnrollment"]["enabled"] = True + with pytest.raises(workspace.WorkspaceError): + workspace.project_workspace(owner, {}, site, CATALOG, lambda _: {}) + + +@pytest.mark.parametrize("mode", ["vm", "container"]) +def test_operator_revision_pin_is_shared_and_does_not_change_repository(mode): + owner, site = inputs() + owner["values"]["deployment"]["mode"] = mode + # Registration fields are not an authority to select source code. + owner["workspaceRevisions"] = {"alice": "c"*40} + owner["values"]["global"] = {"targetRevision": "tenant-branch", "repoURL": "https://foreign.example/repo"} + defaults = {"global": {"repoURL": "https://approved.example/repo", "targetRevision": "site-main"}, + "workspaceRevisions": {"alice": "a"*40, "bob": "b"*40}} + before = copy.deepcopy((owner, defaults, site)) + result = workspace.project_workspace(owner, defaults, site, CATALOG) + assert result["global"] == {"repoURL": defaults["global"]["repoURL"], "targetRevision": "a"*40} + assert "workspaceRevisions" not in result + assert (owner, defaults, site) == before + del defaults["workspaceRevisions"]["alice"] + assert workspace.project_workspace(owner, defaults, site, CATALOG)["global"] == defaults["global"] + + +@pytest.mark.parametrize("pins", [None, [], {"alice": "main"}, {"alice": "a"*39}, + {"alice": "A"*40}, {"alice": {"repoURL": "https://foreign.example/repo", "revision": "a"*40}}, + {"../alice": "a"*40}, {"alice": 123}]) +def test_revision_pin_requires_operator_mapping_of_owner_to_full_commit(pins): + with pytest.raises(workspace.WorkspaceError, match="workspaceRevisions"): + workspace.project_workspace({"name": "alice"}, {"workspaceRevisions": pins}, {}, CATALOG) + + +def test_kata_requires_reviewed_placement_and_reuses_the_same_targets(): + owner, site = inputs() + targets = workspace.project_workspace(owner, {}, site, CATALOG)["containerSites"]["alice"]["targets"] + owner["values"]["deployment"]["mode"] = "kata" + with pytest.raises(workspace.WorkspaceError, match="RuntimeClass"): + workspace.project_workspace(owner, {}, site, CATALOG) + site["kataRuntimeClass"] = {"name": "saw-kata", "uid": owner["installationID"], "sha256": "f"*64} + actual = workspace.project_workspace(owner, {}, site, CATALOG)["containerSites"]["alice"] + assert actual["targets"] == targets + assert actual["kataRuntimeClass"] == site["kataRuntimeClass"] + owner["values"]["deployment"]["mode"] = "container" + with pytest.raises(workspace.WorkspaceError, match="Kata approval"): + workspace.project_workspace(owner, {}, site, CATALOG) diff --git a/values-cluster-mtz2c.yaml b/values-cluster-mtz2c.yaml new file mode 100644 index 000000000..21e75ecf5 --- /dev/null +++ b/values-cluster-mtz2c.yaml @@ -0,0 +1,14 @@ +# Keep owner subjects and runtime identifiers in the cluster Secret. This +# public Pattern values file only names the Secret consumed by RHDH. +clusterGroup: + applications: + openshell-rhdh: + overrides: + - name: portal.containerSitesSecretName + value: saw-portal-container-sites + forceString: true + governance-interceptor: + overrides: + - name: image + value: image-registry.openshift-image-registry.svc:5000/saw-component-builds/governance-interceptor@sha256:0a55721b3753c8895bb695fc2b1895d790c9ce65f1d868e6859b772c620b8c03 + forceString: true