diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..7b474e2 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,22 @@ +[run] +source = . +omit = + tests/* + examples/* + scripts/* + */__pycache__/* + */site-packages/* + setup.py + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise AssertionError + raise NotImplementedError + if __name__ == .__main__.: + if TYPE_CHECKING: + @abstractmethod + +[html] +directory = htmlcov diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..4a29f5d --- /dev/null +++ b/.flake8 @@ -0,0 +1,14 @@ +[flake8] +max-line-length = 127 +max-complexity = 15 +exclude = + .git, + __pycache__, + build, + dist, + .venv, + venv, + scripts/ +ignore = E203, W503 +per-file-ignores = + __init__.py:F401 diff --git a/.github/workflows/build-dockerfiles.yml b/.github/workflows/build-containerfiles.yml similarity index 84% rename from .github/workflows/build-dockerfiles.yml rename to .github/workflows/build-containerfiles.yml index b37acfb..8c1e415 100644 --- a/.github/workflows/build-dockerfiles.yml +++ b/.github/workflows/build-containerfiles.yml @@ -1,4 +1,4 @@ -name: Build All Dockerfiles +name: Build All Containerfiles on: push: @@ -23,18 +23,18 @@ env: REGISTRY: ghcr.io jobs: - build-dockerfiles: - name: Build Docker Images + build-containerfiles: + name: Build Container Images runs-on: ubuntu-latest strategy: matrix: - dockerfile: - - Dockerfile - - Dockerfile.ipxe + containerfile: + - Containerfile + - Containerfile.ipxe include: - - dockerfile: Dockerfile + - containerfile: Containerfile image_postfix: "" - - dockerfile: Dockerfile.ipxe + - containerfile: Containerfile.ipxe image_postfix: "-ipxe" permissions: @@ -62,11 +62,11 @@ jobs: images: ${{ env.REGISTRY }}/${{ github.repository }}${{ matrix.image_postfix }} tags: ${{ inputs.tag_pattern }} - - name: Build and push Docker image + - name: Build and push container image uses: docker/build-push-action@v5 with: context: . - file: ${{ matrix.dockerfile }} + file: ${{ matrix.containerfile }} platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} @@ -85,7 +85,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Run Trivy vulnerability scanner on all Dockerfiles + - name: Run Trivy vulnerability scanner on all Containerfiles uses: aquasecurity/trivy-action@master with: scan-type: 'config' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7b9721..83c85a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - pip install pytest pytest-asyncio black flake8 + pip install -r requirements-dev.txt - name: Lint with flake8 run: | @@ -44,15 +44,22 @@ jobs: - name: Check formatting with black run: | black --check --diff . - - # - name: Test with pytest - # run: | - # pytest -v - build-dockerfiles: - name: Build All Docker Images + - name: Run unit tests + run: | + pytest tests/ -v --cov --cov-report=term-missing --cov-report=xml -m "not integration and not e2e" + + - name: Upload coverage reports + uses: codecov/codecov-action@v3 + with: + files: ./coverage.xml + flags: unittests + name: codecov-nio + + build-containerfiles: + name: Build All Container Images needs: test - uses: ./.github/workflows/build-dockerfiles.yml + uses: ./.github/workflows/build-containerfiles.yml with: tag_pattern: | type=ref,event=branch @@ -78,8 +85,8 @@ jobs: # - name: Deploy to test cluster # run: | - # # Здесь можно добавить команды для деплоя в тестовый кластер - # # Например, с использованием kind или minikube + # # Add commands here for deploying to test cluster + # # For example, using kind or minikube # echo "Deploying to test environment..." # # kubectl apply -f crds/ # # kubectl apply -f deployment.yaml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9876d7c..73f6bed 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,18 +2,18 @@ name: Nightly Build on: schedule: - # Запускается каждый день в 2:00 UTC + # Runs every day at 2:00 UTC - cron: '0 2 * * *' - workflow_dispatch: # Позволяет запускать вручную + workflow_dispatch: # Allows manual triggering env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: - build-dockerfiles: - name: Build All Docker Images for Nightly - uses: ./.github/workflows/build-dockerfiles.yml + build-containerfiles: + name: Build All Container Images for Nightly + uses: ./.github/workflows/build-containerfiles.yml with: tag_pattern: | nightly @@ -48,7 +48,7 @@ jobs: # - name: Run integration tests # run: | - # # Здесь можно добавить интеграционные тесты с Kind + # # Add integration tests with Kind here # echo "Running integration tests..." # # kind create cluster --config kind-config.yaml # # kubectl apply -f crds/ @@ -87,18 +87,18 @@ jobs: notify: name: Notify Status runs-on: ubuntu-latest - needs: [build-dockerfiles, integration-tests, security-scan] + needs: [build-containerfiles, security-scan] if: always() steps: - name: Notify on success - if: needs.build-dockerfiles.result == 'success' && needs.integration-tests.result == 'success' + if: needs.build-containerfiles.result == 'success' && needs.security-scan.result == 'success' run: | echo "Nightly build completed successfully!" - # Здесь можно добавить уведомления (Slack, Discord, etc.) + # Add notifications here (Slack, Discord, etc.) - name: Notify on failure - if: needs.build-dockerfiles.result == 'failure' || needs.integration-tests.result == 'failure' + if: needs.build-containerfiles.result == 'failure' || needs.security-scan.result == 'failure' run: | echo "Nightly build failed! Check the logs." - # Здесь можно добавить уведомления об ошибках + # Add failure notifications here diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d64b20..6368f84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,9 +12,9 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-dockerfiles: - name: Build All Docker Images for Release - uses: ./.github/workflows/build-dockerfiles.yml + build-containerfiles: + name: Build All Container Images for Release + uses: ./.github/workflows/build-containerfiles.yml with: tag_pattern: | type=semver,pattern={{version}} @@ -26,7 +26,7 @@ jobs: generate-manifests: name: Generate Release Manifests runs-on: ubuntu-latest - needs: build-dockerfiles + needs: build-containerfiles permissions: contents: read @@ -36,11 +36,11 @@ jobs: - name: Generate deployment manifests run: | - # Создаем версионированные манифесты для релиза + # Create versioned manifests for release mkdir -p manifests/release cp deployment.yaml manifests/release/ - - # Обновляем версию образа в манифесте для всех Dockerfiles + + # Update image version in manifest for all Containerfiles # Main image sed -i "s|nixos-operator:latest|${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${GITHUB_REF_NAME}|g" manifests/release/deployment.yaml # iPXE image (if used in deployment) @@ -88,7 +88,7 @@ jobs: - name: Update version in README run: | - # Обновляем версию в README если нужно + # Update version in README if needed VERSION=${{ github.event.release.tag_name }} echo "Updating documentation for version $VERSION" - # Здесь можно добавить логику обновления документации + # Add documentation update logic here diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml new file mode 100644 index 0000000..486e0c1 --- /dev/null +++ b/.github/workflows/test-e2e.yml @@ -0,0 +1,72 @@ +name: E2E Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + test-e2e: + name: End-to-End Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Install kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + kind version + + - name: Create kind cluster + run: | + kind create cluster --name nio-e2e --wait 5m + kubectl cluster-info + kubectl wait --for=condition=Ready nodes --all --timeout=5m + + - name: Build operator image + run: | + podman build -t nio-operator:test . + + - name: Load image into kind + run: | + podman save nio-operator:test | kind load image-archive /dev/stdin --name nio-e2e + + - name: Install CRDs + run: | + kubectl apply -f crds/ + + - name: Run E2E tests + run: | + # Run only E2E marked tests + pytest tests/ -v -m e2e --maxfail=5 + + - name: Collect logs on failure + if: failure() + run: | + kubectl get all -A + kubectl describe pods -A + kubectl logs -l app=nio-operator -n nixos-operator-system --tail=100 || true + + - name: Cleanup + if: always() + run: | + kind delete cluster --name nio-e2e || true diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..bf84b6d --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,23 @@ +# Markdownlint configuration for NIO documentation +# +# Technical documentation often requires longer lines for: +# - Command examples with long paths +# - URLs and links +# - Code examples +# - Configuration snippets +# +# This config disables overly restrictive rules while maintaining quality. + +# Disable line length limit - technical docs have long commands/URLs +MD013: false + +# Allow bare URLs in documentation for readability +MD034: false + +# Allow lists and code blocks without surrounding blank lines +# for better readability in technical documentation +MD031: false +MD032: false + +# All other rules enabled with default settings +default: true diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..5f53142 --- /dev/null +++ b/.yamllint @@ -0,0 +1,27 @@ +# yamllint configuration for Kubernetes manifests +# +# Kubernetes YAML uses 2-space indentation and often has long +# description fields for alerts and annotations. + +extends: default + +rules: + # Disable document start requirement (optional in Kubernetes) + document-start: disable + + # Kubernetes standard is 2-space indentation + # Allow consistent 2-space indent for all levels + indentation: + spaces: 2 + indent-sequences: consistent + + # Allow longer lines for descriptions and annotations + # Alert descriptions can be long - increase to 200 + line-length: + max: 200 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: true + + # Allow truthy values (on, off, yes, no) - common in Kubernetes + truthy: + allowed-values: ['true', 'false', 'on', 'off', 'yes', 'no'] diff --git a/Dockerfile b/Containerfile similarity index 81% rename from Dockerfile rename to Containerfile index f91b8f8..8ecbb01 100644 --- a/Dockerfile +++ b/Containerfile @@ -25,10 +25,11 @@ RUN groupadd -g ${USER_GID} operator_group \ ADD https://install.determinate.systems/nix /tmp/nix-installer # Install Nix (will be cached if installer doesn't change) +# SECURITY: Enable sandbox for build isolation RUN chmod +x /tmp/nix-installer \ && /tmp/nix-installer install linux \ - --extra-conf "sandbox = false" \ - --extra-conf "filter-syscalls = false" \ + --extra-conf "sandbox = relaxed" \ + --extra-conf "filter-syscalls = true" \ --init none \ --no-confirm \ && rm -f /tmp/nix-installer @@ -43,13 +44,23 @@ RUN pip install --no-cache-dir -r requirements.txt # Copy only essential runtime files COPY main.py . +COPY config.py . COPY machine_handlers.py . COPY nixosconfiguration_handlers.py . +COPY reconcile_helpers.py . +COPY retry_utils.py . +COPY metrics.py . COPY clients.py . COPY utils.py . COPY events.py . +COPY ssh_utils.py . +COPY known_hosts_manager.py . +COPY input_validation.py . COPY scripts/ ./scripts/ COPY crds/ ./crds/ +# Expose metrics port +EXPOSE 8000 + ENV KUBECONFIG=/app/.kube/config PYTHONUNBUFFERED=1 PYTHONPATH=/app CMD ["python", "main.py"] diff --git a/Dockerfile.ipxe b/Containerfile.ipxe similarity index 100% rename from Dockerfile.ipxe rename to Containerfile.ipxe diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 893fb36..df070d9 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,6 +2,8 @@ This document describes tools and processes for developing and debugging the nixos-operator. +> **Note on Container Tools:** This project uses Podman by default for building OCI images, but you can use any OCI-compatible tool (Docker, Buildah, etc.). All `podman` commands can be replaced with `docker` if preferred. The `docker-compose.yml` file works with both `docker compose` and `podman compose`. + ## 📁 Created Files ### 1. Kind Cluster Setup Script (`kind-setup.sh`) @@ -13,7 +15,7 @@ chmod +x kind-setup.sh ./kind-setup.sh ``` -### 2. Docker Compose for Development (`docker-compose.yml`) +### 2. Compose for Development (`docker-compose.yml`) Starts a complete development environment with Kind cluster and operator in debug mode. **Usage:** @@ -40,9 +42,9 @@ docker-compose logs -f nixos-operator-dev 3. Select "NixOS Operator: Local Debug" 4. Click "Start Debugging" -### Remote Debugging in Docker +### Remote Debugging in Container 1. Start docker-compose: `docker-compose up -d` -2. In VS Code select "NixOS Operator: Remote Debug (Docker)" +2. In VS Code select "NixOS Operator: Remote Debug (Container)" 3. Click "Start Debugging" ### Debugging with Kind Cluster @@ -61,7 +63,7 @@ kubectl apply -f examples/machine-example.yaml kubectl apply -f examples/nixosconfiguration-example.yaml ``` -### Option 2: Development with Docker Compose +### Option 2: Development with Compose ```bash # Start complete development environment docker-compose up -d @@ -106,8 +108,8 @@ kubectl describe nixosconfiguration Available tasks (Ctrl+Shift+P → "Tasks: Run Task"): - `setup-k8s-environment` - configure Kubernetes environment - `setup-kind-cluster` - create Kind cluster -- `start-docker-compose-dev` - start Docker Compose -- `build-operator-image` - build Docker image +- `start-compose-dev` - start Compose +- `build-operator-image` - build container image - `install-dependencies` - install Python dependencies ## 🔍 Monitoring @@ -128,13 +130,13 @@ kubectl logs -f deployment/nixos-operator -n nixos-operator-system | grep -E "(E ## 🛠️ Troubleshooting ### Issue: Kind Cluster Not Creating -**Solution:** Ensure Docker is running and you have permissions to create containers. +**Solution:** Ensure your container runtime (Podman/Docker) is running and you have permissions to create containers. ### Issue: Operator Not Connecting to Kubernetes **Solution:** Check KUBECONFIG configuration and ensure Kind cluster is running. ### Issue: Debugger Not Connecting -**Solution:** Ensure port 5678 is not occupied and Docker Compose is running. +**Solution:** Ensure port 5678 is not occupied and Compose is running. ### Issue: CRDs Not Applying **Solution:** Check access permissions and ensure you're in the correct namespace. @@ -193,8 +195,8 @@ kubectl logs -f deployment/nixos-operator -n nixos-operator-system ### 4. Build and Deploy ```bash -# Build Docker image -docker build -t nixos-operator:latest . +# Build container image (use podman or docker) +podman build -t nixos-operator:latest . # Update deployment kubectl rollout restart deployment/nixos-operator -n nixos-operator-system @@ -235,8 +237,8 @@ pip install -r requirements.txt ### Production Deployment ```bash -# Build production image -docker build -t nixos-operator:latest . +# Build production image (use podman or docker) +podman build -t nixos-operator:latest . # Apply to production cluster kubectl apply -f crds/ @@ -245,8 +247,8 @@ kubectl apply -f deployment.yaml ### Development Deployment ```bash -# Use development setup -docker-compose up -d +# Use development setup (use podman compose or docker compose) +podman compose up -d ``` ## 📚 Additional Resources diff --git a/README.md b/README.md index c2f1c50..7ca530b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A Kubernetes-native operator for declarative management of bare-metal and virtual machines running NixOS. +> **Note:** This project uses Podman by default for building OCI container images, but you can use any OCI-compatible tool (Docker, Buildah, etc.). All `podman` commands in examples can be replaced with `docker` if preferred. + ## Features - **GitOps Approach**: Configurations are managed through Git repositories @@ -63,8 +65,8 @@ spec: # Apply CRDs kubectl apply -f crds/ -# Build and run the operator -docker build -t nixos-operator:latest . +# Build and run the operator (using Podman) +podman build -t nixos-operator:latest . kubectl apply -f deployment.yaml ``` diff --git a/SECURITY_REVIEW.md b/SECURITY_REVIEW.md new file mode 100644 index 0000000..61ba2a6 --- /dev/null +++ b/SECURITY_REVIEW.md @@ -0,0 +1,555 @@ +# Security and Code Quality Review + +**Review Date:** 2025-10-31 +**Reviewer:** Security Audit +**Overall Rating:** 2/10 - NOT PRODUCTION READY + +--- + +## Executive Summary + +This review identifies **27 critical issues** across security, architecture, and code quality domains. The operator has solid conceptual foundations but requires significant remediation before production deployment. + +**Critical Security Issues:** 7 +**Architecture Problems:** 9 +**Code Quality Issues:** 11 + +**Estimated Remediation Time:** 3-4 weeks for a single developer. + +--- + +## 🚨 Critical Security Issues (P0 - Fix Immediately) + +### 1. MITM Vulnerability: Disabled SSH Host Verification + +**Location:** `machine_handlers.py:25`, `nixosconfiguration_handlers.py:224` + +```python +ssh_config = { + "known_hosts": None, # Complete security hole +} +``` + +```python +nix_sshopts = f"-i {tmp_key_path} -o StrictHostKeyChecking=no" # Another hole +``` + +**Impact:** Any network attacker can impersonate target hosts and intercept SSH keys, passwords, and NixOS configurations. + +**Remediation:** +- Implement proper known_hosts management +- Never disable StrictHostKeyChecking +- Add host fingerprint verification + +--- + +### 2. SSH Key Leakage Through Temporary Files + +**Location:** `machine_handlers.py:40-47`, `nixosconfiguration_handlers.py:216-221` + +```python +with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix="_ssh_key") as temp_file: + temp_file.write(secret_data["ssh-privatekey"]) + ssh_key_temp_file = temp_file.name +``` + +**Issues:** +- Keys written to `/tmp` with predictable names +- If process crashes before finally block, keys remain on filesystem +- No O_EXCL flag on file creation (race condition) +- Keys may persist in container overlay filesystem after crash + +**Remediation:** +- Use `asyncssh` without intermediate files (pass key directly) +- If files required, use ramdisk (`/dev/shm` or memory-backed tmpfs) +- Add cleanup via atexit handlers +- Implement secure file creation with O_EXCL + +--- + +### 3. Disabled Nix Isolation + +**Location:** `Containerfile:30-31` + +```dockerfile +--extra-conf "sandbox = false" \ +--extra-conf "filter-syscalls = false" \ +``` + +**Impact:** +- Nix builds can access host filesystem +- Privilege escalation possible via malicious flakes +- Container becomes attack vector against Kubernetes node + +**Remediation:** +- Do not disable sandbox +- If network access needed, use `sandbox = relaxed` instead of `false` +- Enable syscall filtering for defense in depth + +--- + +### 4. Credentials in Logs and Environment Variables + +**Location:** `utils.py:146-151` + +```python +git_kwargs["env"] = {"GIT_SSH_COMMAND": f"ssh -i {ssh_key}"} # Key in env +auth_url = f"{parsed_url.scheme}://token:{secret_data['token']}@{parsed_url.netloc}{parsed_url.path}" # Token in URL +``` + +**Issues:** +- SSH keys visible in environment variables (via `/proc//environ`) +- Tokens in URLs get logged by Git +- Process listing exposes credentials + +**Remediation:** +- Use credential helpers instead of inline credentials +- For SSH, use ssh-agent socket forwarding +- For HTTPS, use git credential helper +- Never log URLs containing credentials + +--- + +### 5. Insecure Temporary File Handling (CWE-377) + +**Location:** Multiple functions creating temp files without proper security + +**Issues:** +- No O_EXCL flag prevents race conditions +- Predictable file paths enable symlink attacks +- World-readable permissions before chmod + +**Remediation:** +- Use `tempfile.mkstemp()` with proper mode parameter +- Set umask before file creation +- Use atomic file operations + +--- + +### 6. Missing Input Validation + +**Location:** Throughout codebase + +**Issues:** +- No validation of `hostname` field (can contain shell metacharacters) +- Git URLs not validated (can contain command injection) +- Flake references not sanitized + +**Example Attack:** +```yaml +spec: + hostname: "target.com; rm -rf /" # Command injection +``` + +**Remediation:** +- Validate all user inputs with strict whitelists +- Use parameterized commands instead of shell strings +- Implement input sanitization for all external data + +--- + +### 7. Insufficient Error Information Disclosure + +**Location:** `clients.py:86`, multiple locations + +```python +except Exception as e: + logger.error(f"Failed to get secret {secret_name}: {e}") + raise # Exposes internal details in error messages +``` + +**Impact:** Error messages may leak sensitive information about infrastructure + +**Remediation:** +- Sanitize error messages before exposing to users +- Log detailed errors internally +- Return generic errors externally + +--- + +## 🏗️ Architecture Problems (P1 - Fix Before Production) + +### 8. Massive Code Duplication + +**Location:** `machine_handlers.py:17-143` vs `machine_handlers.py:146-270` + +SSH connection logic is **COMPLETELY DUPLICATED** between two functions (125+ lines of identical code). + +**Impact:** +- Bugs must be fixed in multiple places +- Inconsistent behavior risk +- Maintenance nightmare + +**Remediation:** Extract common function `establish_ssh_connection()`. + +--- + +### 9. Giant Function with 12 Nesting Levels + +**Location:** `nixosconfiguration_handlers.py:323-554` (232 lines) + +Function `reconcile_nixos_configuration` does EVERYTHING: +- Checks machine availability +- Clones Git repository +- Injects files +- Calculates hashes +- Applies configuration +- Updates statuses +- Removes old versions + +**Impact:** +- Impossible to test individual components +- High cyclomatic complexity +- Difficult to debug + +**Remediation:** Split into 6-8 focused functions with single responsibilities. + +--- + +### 10. Logic Error in Hash Calculation + +**Location:** `nixosconfiguration_handlers.py:158-174` + +```python +files_content = [] +for file_spec in config_spec["additionalFiles"]: + file_info = { # Overwritten each iteration + "path": file_spec.get("path", ""), + ... + } +content_str = json.dumps(file_info, sort_keys=True) # Only LAST file hashed +return hashlib.sha256(content_str.encode("utf-8")).hexdigest() +``` + +**Impact:** Hash only considers last file from `additionalFiles`, all others ignored. + +**Remediation:** Use `files_content.append(file_info)` and hash entire list. + +--- + +### 11. Missing Timeouts on Long Operations + +`nixos-rebuild` and `nixos-anywhere` can run for hours. No timeouts anywhere: + +```python +process = await asyncio.create_subprocess_shell(cmd, ...) # No timeout +await process.wait() # Can hang forever +``` + +**Impact:** +- Hung processes accumulate +- Memory/resource leaks +- Impossible graceful shutdown + +**Remediation:** Add `asyncio.wait_for()` with reasonable timeouts (30-60 minutes). + +--- + +### 12. No Retry Logic with Exponential Backoff + +On transient network issues, operator immediately fails: + +```python +except Exception as e: + raise kopf.TemporaryError(f"...", delay=60) # Fixed 60s delay +``` + +**Remediation:** Exponential backoff with jitter (60s → 120s → 240s → ...). + +--- + +### 13. Race Condition in Git Operations + +**Location:** `nixosconfiguration_handlers.py:113-119` + +```python +subprocess.run(["git", "add", "--intent-to-add", rel_path], ...) +``` + +**Issues:** +- Temporary files added to git index +- Parallel reconcile runs cause state corruption +- `--intent-to-add` doesn't commit but leaves files in index + +**Remediation:** Use worktrees or avoid touching git index. + +--- + +### 14. No Graceful Shutdown + +**Location:** `main.py:96` + +```python +if __name__ == "__main__": + kopf.run() # No signal handlers +``` + +**Impact:** On rolling update in Kubernetes: +- Interrupted SSH sessions +- Incomplete git operations +- Corrupted state in CRD status + +**Remediation:** Add proper shutdown handlers with draining active reconciles. + +--- + +### 15. Resource Leaks + +**Location:** `nixosconfiguration_handlers.py:550` + +```python +finally: + shutil.rmtree(repo_path, ignore_errors=True) # Hides problems +``` + +If `shutil.rmtree` fails, directories accumulate in `/tmp`. + +**Remediation:** +- Log errors instead of ignoring +- Add periodic cleanup job +- Monitor disk usage + +--- + +### 16. Inefficient Git Operations + +**Location:** `utils.py:189-192` + +```python +origin.fetch(**git_kwargs) # FULL FETCH +for ref_info in repo.git.ls_remote(git_url, ref).split("\n"): # Second request +``` + +**Impact:** Unnecessary network traffic and latency + +**Remediation:** `git ls-remote` doesn't require fetch, make single request. + +--- + +## 💩 Code Quality Issues (P2 - Fix for Maintainability) + +### 17. Mixed Languages in Code + +**Location:** `main.py:28`, `main.py:41`, `main.py:52` + +```python +"""Обработчик создания Machine""" # Russian +"""Периодическая проверка доступности машин""" # Russian +``` + +**Standard:** All code, comments, and documentation must be in English. + +--- + +### 18. print() Instead of logging + +**Location:** `main.py:15`, `clients.py:18-20` + +```python +print("starting") # Wrong +print(f"Attempting to connect to Kubernetes") # Wrong +``` + +**Remediation:** Use `logger.info()` everywhere. + +--- + +### 19. Emojis in Production Logs + +**Location:** `clients.py:35-38`, `nixosconfiguration_handlers.py:46,105` + +```python +logger.info("✅ Successfully loaded kubeconfig") +logger.warning(f"❌ Failed to load kubeconfig: {e}") +# 👈 Store paths of injected files +``` + +**Issues:** +- Breaks grep/awk log parsing +- Encoding issues in some terminals +- Unprofessional appearance + +**Remediation:** Remove ALL emojis from code. + +--- + +### 20. Dead Code + +**Location:** `clients.py:106-108` + +```python +else: + # For creating status + pass # Useless block +``` + +**Remediation:** Remove. + +--- + +### 21. Unused Dependencies + +**Location:** `requirements.txt:6` + +``` +asyncio # Part of stdlib, not needed in requirements +``` + +**Location:** `Containerfile:6` +```dockerfile +kubectl # Never used in code +``` + +**Remediation:** Remove unused dependencies. + +--- + +### 22. Imports Inside Functions + +**Location:** `main.py:68`, `nixosconfiguration_handlers.py:245` + +```python +from datetime import datetime # Should be at file top +from scripts.facts_parser import parse_facts # Should be at file top +``` + +**Remediation:** Move all imports to file beginning. + +--- + +### 23. Hardcoded Values Everywhere + +```python +base_path = "/tmp/nixos-config" # utils.py:20 - No env variable +interval=120 # main.py:84 - Cannot configure +interval=300.0 # main.py:52 - Cannot configure +``` + +**Remediation:** Configure via environment variables or ConfigMap. + +--- + +### 24. Missing Type Hints + +```python +def get_machine(machine_name: str, namespace: str): # No return type + return custom_objects_api.get_namespaced_custom_object(...) +``` + +**Remediation:** Add type hints everywhere (Python 3.11+ supported). + +--- + +### 25. No Tests + +Project has **ZERO TESTS**. This is a production operator for infrastructure management. + +**Remediation:** +- Unit tests for all utility functions +- Integration tests for Kubernetes API interactions +- E2E tests for SSH operations (with mock SSH server) + +--- + +### 26. No Metrics + +Operator exports zero metrics: +- Machines in each state +- Configurations applied successfully/with errors +- Operation latency +- Error rate + +**Remediation:** Add Prometheus metrics. + +--- + +### 27. Poor Error Messages + +```python +except Exception as e: + logger.error(f"Failed to reconcile NixosConfiguration {name}: {e}") # Loses traceback +``` + +**Remediation:** Use `exc_info=True` everywhere to preserve stack traces. + +--- + +## ✅ What's Done Well + +1. **Using Kopf** - excellent choice for Kubernetes operators +2. **Async/await** - correct approach for I/O operations +3. **Status subresources** - proper Kubernetes API usage +4. **Git-based configuration** - solid GitOps approach +5. **Project structure** - logical module separation + +--- + +## 🎯 Final Assessment + +**Concept:** 8/10 - excellent idea +**Implementation:** 2/10 - multiple critical issues +**Security:** 1/10 - FAIL +**Code Quality:** 3/10 - needs serious refactoring +**Production Readiness:** 0/10 - NOT READY + +--- + +## 📋 Remediation Roadmap + +### Phase 1: Critical Security Fixes (Week 1) +- [ ] Enable SSH host verification (#1) +- [ ] Fix credential leakage (#2, #4) +- [ ] Enable Nix sandbox (#3) +- [ ] Add input validation (#6) +- [ ] Fix hash calculation bug (#10) + +### Phase 2: Architecture Improvements (Week 2) +- [ ] Add timeouts (#11) +- [ ] Implement retry logic (#12) +- [ ] Refactor duplicated code (#8) +- [ ] Split giant function (#9) +- [ ] Add graceful shutdown (#14) + +### Phase 3: Code Quality (Week 3) +- [ ] Remove emojis and Russian comments (#17, #19) +- [ ] Add type hints (#24) +- [ ] Fix hardcoded values (#23) +- [ ] Move imports to top (#22) +- [ ] Clean up dead code (#20) + +### Phase 4: Testing and Observability (Week 4) +- [ ] Add unit tests (#25) +- [ ] Add integration tests +- [ ] Implement metrics (#26) +- [ ] Improve error messages (#27) +- [ ] Add E2E tests + +--- + +## 🔐 Security Compliance + +**Current Status:** +- ❌ CWE-377: Insecure Temporary File +- ❌ CWE-259: Use of Hard-coded Credentials +- ❌ CWE-78: OS Command Injection +- ❌ CWE-200: Information Exposure +- ❌ CWE-327: Use of Broken Crypto (disabled verification) + +**Required Before Production:** +- Security audit by independent third party +- Penetration testing +- SAST/DAST scanning +- Dependency vulnerability scanning + +--- + +## 📚 References + +- [OWASP Top 10](https://owasp.org/www-project-top-ten/) +- [CWE Top 25](https://cwe.mitre.org/top25/) +- [Kubernetes Security Best Practices](https://kubernetes.io/docs/concepts/security/) +- [Python Security Best Practices](https://python.readthedocs.io/en/latest/library/security_warnings.html) + +--- + +**Recommendation:** Address all P0 and P1 issues before ANY production deployment. This code in its current state poses significant security risks. diff --git a/clients.py b/clients.py index 1d83220..48886a3 100644 --- a/clients.py +++ b/clients.py @@ -4,20 +4,21 @@ import kubernetes import logging import sys -from typing import Dict import os +from typing import Dict, Any from pathlib import Path + logger = logging.getLogger(__name__) -def setup_kubernetes_client(): +def setup_kubernetes_client() -> None: kubeconfig_path = os.environ.get("KUBECONFIG", "~/.kube/config") expanded_kubeconfig = os.path.expanduser(kubeconfig_path) kubeconfig_file = Path(expanded_kubeconfig) - print(f"Attempting to connect to Kubernetes") - print(f"KUBECONFIG variable: {kubeconfig_path}") - print(f"Expanded kubeconfig path: {expanded_kubeconfig}") + logger.info(f"Attempting to connect to Kubernetes") + logger.info(f"KUBECONFIG variable: {kubeconfig_path}") + logger.info(f"Expanded kubeconfig path: {expanded_kubeconfig}") # Check if file exists if kubeconfig_file.exists(): @@ -32,12 +33,12 @@ def setup_kubernetes_client(): # Try to load kubeconfig try: kubernetes.config.load_kube_config(config_file=expanded_kubeconfig) - logger.info("✅ Successfully loaded kubeconfig") + logger.info("Successfully loaded kubeconfig") return except kubernetes.config.ConfigException as e: - logger.warning(f"❌ Failed to load kubeconfig: {e}") + logger.warning(f"Failed to load kubeconfig: {e}") except Exception as e: - logger.error(f"❗ Unexpected error loading kubeconfig: {e}", exc_info=True) + logger.error(f"Unexpected error loading kubeconfig: {e}", exc_info=True) else: logger.warning(f"Kubeconfig file NOT found: {kubeconfig_file}") @@ -51,16 +52,16 @@ def setup_kubernetes_client(): logger.info(f"KUBERNETES_SERVICE_PORT: {port}") if not host or not port: - logger.error("❌ KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT variables not set — in-cluster config impossible") + logger.error("KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT variables not set - in-cluster config impossible") try: kubernetes.config.load_incluster_config() - logger.info("✅ Successfully loaded in-cluster config") + logger.info("Successfully loaded in-cluster config") except kubernetes.config.ConfigException as e: - logger.error(f"❌ In-cluster connection error: {e}") + logger.error(f"In-cluster connection error: {e}") sys.exit(1) except Exception as e: - logger.error(f"❗ Critical error during in-cluster connection: {e}", exc_info=True) + logger.error(f"Critical error during in-cluster connection: {e}", exc_info=True) sys.exit(1) # Call initialization @@ -88,8 +89,8 @@ async def get_secret_data(secret_name: str, namespace: str) -> Dict[str, str]: async def update_machine_status( - machine_name: str, namespace: str, status_updates: Dict, patch: bool = True -): + machine_name: str, namespace: str, status_updates: Dict[str, Any], patch: bool = True +) -> None: """Update Machine resource status""" try: body = {"status": status_updates} @@ -103,18 +104,15 @@ async def update_machine_status( name=machine_name, body=body, ) - else: - # For creating status - pass except Exception as e: - logger.error(f"Failed to update machine status: {e}") + logger.error(f"Failed to update machine status: {e}", exc_info=True) raise async def update_configuration_status( - config_name: str, namespace: str, status_updates: Dict -): + config_name: str, namespace: str, status_updates: Dict[str, Any] +) -> None: """Update NixosConfiguration resource status""" try: body = {"status": status_updates} @@ -129,11 +127,11 @@ async def update_configuration_status( ) except Exception as e: - logger.error(f"Failed to update configuration status: {e}") + logger.error(f"Failed to update configuration status: {e}", exc_info=True) raise -def get_machine(machine_name: str, namespace: str): +def get_machine(machine_name: str, namespace: str) -> Dict[str, Any]: """Get Machine resource""" return custom_objects_api.get_namespaced_custom_object( group="nio.homystack.com", diff --git a/config.py b/config.py new file mode 100644 index 0000000..1ce8fd3 --- /dev/null +++ b/config.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +""" +Configuration module for NixOS Infrastructure Operator. + +All configuration values are loaded from environment variables with sensible defaults. +This eliminates hardcoded values and allows runtime configuration via ConfigMaps/Secrets. +""" + +import os +from typing import Optional + + +def get_env_int(key: str, default: int) -> int: + """Get integer value from environment variable.""" + value = os.environ.get(key) + if value is None: + return default + try: + return int(value) + except ValueError: + raise ValueError(f"Environment variable {key}={value} is not a valid integer") + + +def get_env_float(key: str, default: float) -> float: + """Get float value from environment variable.""" + value = os.environ.get(key) + if value is None: + return default + try: + return float(value) + except ValueError: + raise ValueError(f"Environment variable {key}={value} is not a valid float") + + +def get_env_str(key: str, default: str) -> str: + """Get string value from environment variable.""" + return os.environ.get(key, default) + + +# Filesystem paths +BASE_CONFIG_PATH = get_env_str("NIO_BASE_CONFIG_PATH", "/tmp/nixos-config") +KNOWN_HOSTS_PATH = get_env_str("NIO_KNOWN_HOSTS_PATH", "/tmp/nio-ssh-known-hosts") +REMOTE_HARDWARE_SCRIPT_PATH = get_env_str( + "NIO_REMOTE_HARDWARE_SCRIPT_PATH", "/tmp/hardware_scanner.sh" +) + +# Reconciliation intervals (seconds) +MACHINE_DISCOVERY_INTERVAL = get_env_float("NIO_MACHINE_DISCOVERY_INTERVAL", 60.0) +HARDWARE_SCAN_INTERVAL = get_env_float("NIO_HARDWARE_SCAN_INTERVAL", 300.0) +CONFIG_RECONCILE_INTERVAL = get_env_float("NIO_CONFIG_RECONCILE_INTERVAL", 120.0) + +# Operation timeouts (seconds) +NIXOS_APPLY_TIMEOUT = get_env_int("NIO_NIXOS_APPLY_TIMEOUT", 3600) + +# Retry configuration +RETRY_MAX_ATTEMPTS = get_env_int("NIO_RETRY_MAX_ATTEMPTS", 3) +RETRY_INITIAL_DELAY = get_env_float("NIO_RETRY_INITIAL_DELAY", 2.0) +RETRY_MAX_DELAY = get_env_float("NIO_RETRY_MAX_DELAY", 30.0) +RETRY_EXPONENTIAL_BASE = get_env_float("NIO_RETRY_EXPONENTIAL_BASE", 2.0) + +# Metrics and health checks +METRICS_PORT = get_env_int("METRICS_PORT", 8000) +HEALTH_CHECK_PORT = get_env_int("HEALTH_CHECK_PORT", 8080) + + +def get_config_summary() -> str: + """Return configuration summary for logging.""" + return f"""NixOS Infrastructure Operator Configuration: + Paths: + - Base config path: {BASE_CONFIG_PATH} + - Known hosts path: {KNOWN_HOSTS_PATH} + - Remote hardware script: {REMOTE_HARDWARE_SCRIPT_PATH} + + Intervals: + - Machine discovery: {MACHINE_DISCOVERY_INTERVAL}s + - Hardware scan: {HARDWARE_SCAN_INTERVAL}s + - Config reconcile: {CONFIG_RECONCILE_INTERVAL}s + + Timeouts: + - NixOS apply: {NIXOS_APPLY_TIMEOUT}s + + Retry: + - Max attempts: {RETRY_MAX_ATTEMPTS} + - Initial delay: {RETRY_INITIAL_DELAY}s + - Max delay: {RETRY_MAX_DELAY}s + - Exponential base: {RETRY_EXPONENTIAL_BASE} + + Observability: + - Metrics port: {METRICS_PORT} + - Health check port: {HEALTH_CHECK_PORT} +""" diff --git a/deployment.yaml b/deployment.yaml index 69d1ecf..7587fc0 100644 --- a/deployment.yaml +++ b/deployment.yaml @@ -54,7 +54,33 @@ spec: containers: - name: operator image: ghcr.io/homystack/nio:main - imagePullPolicy: IfNotPresent # ←←← ЭТО ОБЯЗАТЕЛЬНО ДЛЯ :latest + imagePullPolicy: IfNotPresent # Required for :latest tag + + ports: + - name: metrics + containerPort: 8000 + protocol: TCP + - name: health + containerPort: 8080 + protocol: TCP + + livenessProbe: + httpGet: + path: /live + port: health + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /ready + port: health + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 env: - name: KUBERNETES_SERVICE_HOST @@ -75,3 +101,37 @@ spec: capabilities: drop: - ALL +--- +apiVersion: v1 +kind: Service +metadata: + name: nixos-operator-metrics + namespace: nixos-operator-system + labels: + app: nixos-operator +spec: + type: ClusterIP + selector: + app: nixos-operator + ports: + - name: metrics + port: 8000 + targetPort: metrics + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: nixos-operator-health + namespace: nixos-operator-system + labels: + app: nixos-operator +spec: + type: ClusterIP + selector: + app: nixos-operator + ports: + - name: health + port: 8080 + targetPort: health + protocol: TCP diff --git a/docker-compose.yml b/docker-compose.yml index aa6553f..9e6563f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: # image: ghcr.io/homystack/nio:main build: context: . - dockerfile: Dockerfile + dockerfile: Containerfile container_name: nixos-operator-dev network_mode: host volumes: @@ -25,7 +25,7 @@ services: nio-ipxe: build: context: . - dockerfile: Dockerfile.ipxe + dockerfile: Containerfile.ipxe container_name: ipxe-server network_mode: host volumes: diff --git a/docs/production-deployment.md b/docs/production-deployment.md new file mode 100644 index 0000000..8bc2d38 --- /dev/null +++ b/docs/production-deployment.md @@ -0,0 +1,649 @@ +# Production Deployment Guide + +This guide covers deploying the NixOS Infrastructure Operator in a production Kubernetes environment with full observability and security. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Deployment Steps](#deployment-steps) +- [Monitoring Setup](#monitoring-setup) +- [Security Considerations](#security-considerations) +- [Resource Planning](#resource-planning) +- [Troubleshooting](#troubleshooting) +- [Operational Best Practices](#operational-best-practices) + +## Prerequisites + +### Kubernetes Cluster Requirements + +- Kubernetes 1.24+ +- CNI plugin configured (Calico, Cilium, or similar) +- StorageClass for persistent volumes (if needed for config storage) +- Network connectivity to target machines via SSH + +### Required Cluster Components + +1. **Prometheus Operator** (recommended for monitoring) + ```bash + kubectl apply --filename https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/bundle.yaml + ``` + +2. **Grafana** (recommended for dashboards) + ```bash + kubectl apply --filename https://raw.githubusercontent.com/grafana/grafana/main/deploy/kubernetes/grafana.yaml + ``` + +### External Dependencies + +- **SSH Access**: Operator needs SSH access to target NixOS machines +- **Git Repositories**: Access to configuration Git repositories (GitHub, GitLab, etc.) +- **Container Registry Access**: Access to `ghcr.io/homystack/nio` images + +## Deployment Steps + +### Step 1: Create Namespace + +```bash +kubectl create namespace nixos-operator-system +``` + +### Step 2: Configure SSH Credentials + +Create a Secret with SSH private key for accessing target machines: + +```bash +kubectl create secret generic nixos-ssh-key \ + --from-file=ssh-privatekey=/path/to/ssh/key \ + --namespace nixos-operator-system +``` + +Mount this secret in the operator Deployment by adding to `deployment.yaml`: + +```yaml +spec: + template: + spec: + volumes: + - name: ssh-key + secret: + secretName: nixos-ssh-key + defaultMode: 0600 + containers: + - name: operator + volumeMounts: + - name: ssh-key + mountPath: /etc/ssh-key + readOnly: true + env: + - name: SSH_KEY_PATH + value: /etc/ssh-key/ssh-privatekey +``` + +### Step 3: Apply Custom Resource Definitions + +```bash +kubectl apply --filename crds/ +``` + +Verify CRDs are created: + +```bash +kubectl get crds | grep nio.homystack.com +``` + +Expected output: + +```text +machines.nio.homystack.com +nixosconfigurations.nio.homystack.com +``` + +### Step 4: Deploy Operator + +```bash +kubectl apply --filename deployment.yaml +``` + +This creates: + +- ServiceAccount with RBAC permissions +- Deployment with operator pod +- Service for Prometheus metrics (port 8000) +- Service for health checks (port 8080) + +### Step 5: Verify Deployment + +Check operator pod is running: + +```bash +kubectl get pods --namespace nixos-operator-system +``` + +Check operator logs: + +```bash +kubectl logs --namespace nixos-operator-system deployment/nixos-operator --follow +``` + +Look for these initialization messages: + +```console +INFO:__main__:NixOS Infrastructure Operator starting +INFO:__main__:Prometheus metrics server started on port 8000 +INFO:__main__:Health check server started on port 8080 +INFO:kopf.objects:Handler 'configure' succeeded. +``` + +### Step 6: Verify Health Endpoints + +Test health endpoints: + +```bash +# Port-forward health service +kubectl port-forward --namespace nixos-operator-system service/nixos-operator-health 8080:8080 + +# In another terminal, test endpoints +curl http://localhost:8080/health # Should return {"status": "healthy"} +curl http://localhost:8080/ready # Should return {"status": "ready"} +curl http://localhost:8080/live # Should return {"status": "alive"} +``` + +## Monitoring Setup + +### Step 1: Deploy ServiceMonitor + +For Prometheus Operator integration: + +```bash +kubectl apply --filename monitoring/service-monitor.yaml +``` + +This configures Prometheus to scrape metrics from the operator automatically. + +### Step 2: Deploy Alerting Rules + +```bash +kubectl apply --filename monitoring/prometheus-rules.yaml +``` + +This creates PrometheusRule with alerts for: + +- Operator health and readiness +- High reconciliation failure rates +- SSH connection failures +- NixOS build failures +- High error rates + +### Step 3: Import Grafana Dashboard + +1. Access Grafana UI +2. Navigate to **Dashboards > Import** +3. Upload `monitoring/grafana/nio-operator-dashboard.json` +4. Select Prometheus datasource +5. Click **Import** + +The dashboard provides: +- Overview of managed machines and configurations +- Reconciliation performance metrics +- SSH and Git operation metrics +- NixOS build duration and success rates +- Error and retry tracking + +### Step 4: Configure Alertmanager + +Add AlertManager route for operator alerts: + +```yaml +route: + receiver: 'default' + routes: + - match: + alertname: NixOSOperatorDown + receiver: 'critical-alerts' + continue: true + - match: + component: operator + receiver: 'operator-team' +``` + +## Security Considerations + +### 1. SSH Key Management + +**Best Practices:** +- Use dedicated SSH keys per cluster/environment +- Rotate SSH keys periodically (recommended: every 90 days) +- Use read-only keys where possible +- Enable SSH key passphrase protection if operator supports it + +**Avoid:** +- Sharing SSH keys across environments +- Using personal SSH keys in production +- Storing unencrypted keys in version control + +### 2. Network Security + +**Firewall Rules:** +```bash +# Allow operator to SSH to target machines +iptables -A OUTPUT -p tcp --dport 22 -j ACCEPT + +# Allow Prometheus to scrape metrics +iptables -A INPUT -p tcp --dport 8000 -j ACCEPT + +# Allow health check probes +iptables -A INPUT -p tcp --dport 8080 -j ACCEPT +``` + +**NetworkPolicy (recommended):** +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: nixos-operator-netpol + namespace: nixos-operator-system +spec: + podSelector: + matchLabels: + app: nixos-operator + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: monitoring + ports: + - protocol: TCP + port: 8000 # Metrics + - protocol: TCP + port: 8080 # Health checks + egress: + - to: + - namespaceSelector: {} + ports: + - protocol: TCP + port: 443 # Kubernetes API + - to: [] # Allow all egress for SSH to external machines + ports: + - protocol: TCP + port: 22 +``` + +### 3. RBAC Permissions + +The operator requires these minimum permissions: + +```yaml +rules: +- apiGroups: ["nio.homystack.com"] + resources: ["machines", "nixosconfigurations"] + verbs: ["get", "list", "watch", "update", "patch"] +- apiGroups: ["nio.homystack.com"] + resources: ["machines/status", "nixosconfigurations/status"] + verbs: ["update", "patch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] # For SSH keys +- apiGroups: [""] + resources: ["events"] + verbs: ["create"] # For Kubernetes events +``` + +**Security hardening:** +- Do not grant cluster-admin permissions +- Use namespaced Roles instead of ClusterRoles if possible +- Audit RBAC permissions regularly + +### 4. Pod Security + +The deployment includes security context: + +```yaml +securityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false # Required for /tmp writes +``` + +**Pod Security Standards:** +- Enable **Restricted** Pod Security Standard where possible +- Consider **Baseline** if operator requires specific capabilities + +## Resource Planning + +### CPU and Memory Requirements + +**Minimum (for testing/development):** +```yaml +resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" +``` + +**Recommended (production):** +```yaml +resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "1000m" +``` + +**Sizing Guidelines:** +- Base memory: 128Mi +- Per managed machine: +20Mi +- Per active reconciliation: +50Mi +- NixOS build operations can spike CPU usage significantly + +Example: Managing 20 machines with 5 concurrent reconciliations: +- Memory: 128 + (20 × 20) + (5 × 50) = 778Mi → round to 1Gi limit +- CPU: 250m base + bursting to 1000m for builds + +### High Availability + +For production HA setup: + +```yaml +spec: + replicas: 1 # Operator uses leader election, multiple replicas supported + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 +``` + +**Note:** Kopf framework handles leader election automatically. Multiple replicas can be deployed for HA, but only one will be active (leader) at a time. + +### Storage Considerations + +The operator uses `/tmp` for: +- Git repository clones +- SSH known_hosts storage +- Temporary configuration files + +**Recommendations:** +- Mount emptyDir volume for `/tmp` with size limit +- Use tmpfs for better performance (ephemeral data) + +```yaml +volumes: +- name: tmp + emptyDir: + sizeLimit: 1Gi + medium: Memory # Optional: use tmpfs +containers: +- name: operator + volumeMounts: + - name: tmp + mountPath: /tmp +``` + +## Troubleshooting + +### Common Issues + +#### 1. Operator Pod CrashLoopBackOff + +**Symptoms:** +```bash +kubectl get pods -n nixos-operator-system +# NAME READY STATUS RESTARTS +# nixos-operator-7d8f9c5b6d-abc12 0/1 CrashLoopBackOff 5 +``` + +**Diagnosis:** +```bash +kubectl logs -n nixos-operator-system nixos-operator-7d8f9c5b6d-abc12 --previous +``` + +**Common causes:** +- Missing SSH key secret +- Invalid Kubernetes API permissions +- Python dependency errors + +**Resolution:** +- Verify SSH secret exists: `kubectl get secret nixos-ssh-key -n nixos-operator-system` +- Check RBAC: `kubectl auth can-i list machines.nio.homystack.com --as=system:serviceaccount:nixos-operator-system:nixos-operator` +- Review image version and dependencies + +#### 2. Machines Not Discoverable + +**Symptoms:** +- Metric `nio_machines_discoverable == 0` +- Logs show SSH connection failures + +**Diagnosis:** +```bash +# Check SSH connectivity from operator pod +kubectl exec -n nixos-operator-system deployment/nixos-operator -- ssh -i /etc/ssh-key/ssh-privatekey user@target-machine +``` + +**Common causes:** +- Network firewall blocking port 22 +- SSH key not authorized on target machine +- Target machine powered off or unreachable + +**Resolution:** +- Verify network connectivity +- Add SSH public key to `~/.ssh/authorized_keys` on target +- Check target machine status and network configuration + +#### 3. High Reconciliation Failure Rate + +**Symptoms:** +- Alert: `HighReconciliationFailureRate` +- Metrics show `nio_configurations_failed_total` increasing + +**Diagnosis:** +```bash +# Check reconciliation error metrics +kubectl port-forward -n nixos-operator-system service/nixos-operator-metrics 8000:8000 +curl http://localhost:8000/metrics | grep nio_reconcile_errors_total +``` + +**Common causes:** +- Invalid NixOS configurations +- Git repository access issues +- NixOS build failures on target machines + +**Resolution:** +- Review operator logs for specific error messages +- Validate NixOS configuration syntax +- Check Git repository access and credentials +- Verify target machine has sufficient resources for builds + +#### 4. Prometheus Not Scraping Metrics + +**Symptoms:** +- Grafana dashboard shows "No data" +- Prometheus targets page shows operator as "Down" + +**Diagnosis:** +```bash +# Verify metrics endpoint is accessible +kubectl port-forward -n nixos-operator-system service/nixos-operator-metrics 8000:8000 +curl http://localhost:8000/metrics +``` + +**Common causes:** +- ServiceMonitor not created or misconfigured +- Prometheus not configured to discover ServiceMonitors +- Network policy blocking Prometheus + +**Resolution:** +- Verify ServiceMonitor: `kubectl get servicemonitor -n nixos-operator-system` +- Check Prometheus operator logs +- Review NetworkPolicy configuration + +### Debug Mode + +Enable verbose logging: + +```yaml +env: +- name: LOG_LEVEL + value: "DEBUG" +``` + +This will log detailed information about: +- SSH connection attempts +- Git clone operations +- NixOS build commands +- Reconciliation decisions + +## Operational Best Practices + +### 1. Backup and Disaster Recovery + +**What to back up:** +- Custom Resource definitions (CRDs) +- Machine and NixOSConfiguration resources +- SSH keys (encrypted) +- Monitoring configuration + +**Backup commands:** +```bash +# Backup all custom resources +kubectl get machines,nixosconfigurations -A -o yaml > nio-resources-backup.yaml + +# Backup monitoring configuration +kubectl get servicemonitor,prometheusrule -n nixos-operator-system -o yaml > nio-monitoring-backup.yaml +``` + +### 2. Rolling Updates + +When updating the operator: + +```bash +# Apply new deployment +kubectl apply --filename deployment.yaml + +# Watch rollout +kubectl rollout status deployment/nixos-operator -n nixos-operator-system + +# Rollback if needed +kubectl rollout undo deployment/nixos-operator -n nixos-operator-system +``` + +**Zero-downtime updates:** +- Operator uses leader election (multiple replicas safe) +- Active reconciliations complete before pod termination +- 5-second grace period configured in cleanup handler + +### 3. Monitoring and Alerting + +**Key metrics to monitor:** +- `nio_machines_total` - Total managed machines +- `nio_machines_discoverable` - Reachable machines +- `rate(nio_configurations_applied_total[5m])` - Success rate +- `rate(nio_configurations_failed_total[5m])` - Failure rate +- `nio_reconcile_duration_seconds` - Performance + +**Critical alerts:** +- `NixOSOperatorDown` - Operator unavailable +- `HighReconciliationFailureRate` - Configuration issues +- `SSHConnectionsCompletelyFailing` - Network/access problems + +### 4. Scaling Considerations + +**Horizontal scaling:** +- Operator supports multiple replicas with leader election +- Only active leader performs reconciliations +- Standby replicas ready for failover + +**Vertical scaling:** +- Increase CPU limits for faster NixOS builds +- Increase memory for managing more machines +- Monitor resource usage and adjust + +**Performance tuning:** +```yaml +env: +- name: NIO_MACHINE_DISCOVERY_INTERVAL + value: "60" # Seconds between discovery checks +- name: NIO_CONFIG_RECONCILE_INTERVAL + value: "120" # Seconds between config reconciliation +- name: NIO_RETRY_MAX_ATTEMPTS + value: "3" +``` + +### 5. Logging and Auditing + +**Centralized logging:** +- Ship logs to Loki, Elasticsearch, or CloudWatch +- Retain logs for compliance requirements +- Enable structured logging for parsing + +**Audit trail:** +- Kubernetes Events created for major operations +- Metrics track all operations with labels +- Git commits provide configuration change history + +### 6. Security Auditing + +**Regular security tasks:** +- Rotate SSH keys every 90 days +- Review RBAC permissions quarterly +- Scan container images for vulnerabilities +- Update dependencies regularly + +**Security scanning:** +```bash +# Scan operator image for vulnerabilities +trivy image ghcr.io/homystack/nio:main + +# Check for outdated Python dependencies +kubectl exec -n nixos-operator-system deployment/nixos-operator -- pip list --outdated +``` + +## Configuration Reference + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `METRICS_PORT` | `8000` | Prometheus metrics HTTP port | +| `HEALTH_CHECK_PORT` | `8080` | Health check endpoints HTTP port | +| `NIO_BASE_CONFIG_PATH` | `/tmp/nixos-config` | Base path for configuration storage | +| `NIO_KNOWN_HOSTS_PATH` | `/tmp/nio-ssh-known-hosts` | SSH known_hosts file path | +| `NIO_MACHINE_DISCOVERY_INTERVAL` | `60.0` | Machine discovery interval (seconds) | +| `NIO_HARDWARE_SCAN_INTERVAL` | `300.0` | Hardware scan interval (seconds) | +| `NIO_CONFIG_RECONCILE_INTERVAL` | `120.0` | Configuration reconciliation interval (seconds) | +| `NIO_NIXOS_APPLY_TIMEOUT` | `3600` | NixOS apply operation timeout (seconds) | +| `NIO_RETRY_MAX_ATTEMPTS` | `3` | Maximum retry attempts for operations | +| `NIO_RETRY_INITIAL_DELAY` | `2.0` | Initial retry delay (seconds) | +| `NIO_RETRY_MAX_DELAY` | `30.0` | Maximum retry delay (seconds) | +| `NIO_RETRY_EXPONENTIAL_BASE` | `2.0` | Exponential backoff base multiplier | + +### Service Ports + +| Port | Protocol | Purpose | +|------|----------|---------| +| 8000 | HTTP | Prometheus metrics endpoint (`/metrics`) | +| 8080 | HTTP | Health check endpoints (`/health`, `/ready`, `/live`) | + +## Support and Resources + +- **Documentation**: https://github.com/homystack/NIO/tree/main/docs +- **Issues**: https://github.com/homystack/NIO/issues +- **Source Code**: https://github.com/homystack/NIO + +For production support, please open an issue with: +- Operator version +- Kubernetes version and platform +- Relevant logs and metrics +- Steps to reproduce the issue diff --git a/events.py b/events.py index 5da6a73..d786a12 100644 --- a/events.py +++ b/events.py @@ -2,11 +2,12 @@ import kopf import logging +from typing import Any, Dict logger = logging.getLogger(__name__) -def emit_missing_credentials_event(body, reason: str, message: str): +def emit_missing_credentials_event(body: Dict[str, Any], reason: str, message: str) -> None: """Create event about missing credentials""" try: kopf.warn(body, reason=reason, message=message) @@ -15,7 +16,7 @@ def emit_missing_credentials_event(body, reason: str, message: str): logger.error(f"Failed to emit missing credentials event: {e}") -def emit_configuration_applied_event(body, reason: str, message: str): +def emit_configuration_applied_event(body: Dict[str, Any], reason: str, message: str) -> None: """Create event about configuration application""" try: kopf.info(body, reason=reason, message=message) @@ -24,7 +25,7 @@ def emit_configuration_applied_event(body, reason: str, message: str): logger.error(f"Failed to emit configuration applied event: {e}") -def emit_error_event(body, reason: str, message: str): +def emit_error_event(body: Dict[str, Any], reason: str, message: str) -> None: """Create error event""" try: kopf.exception(body, reason=reason, message=message) diff --git a/health.py b/health.py new file mode 100644 index 0000000..d68c1c9 --- /dev/null +++ b/health.py @@ -0,0 +1,117 @@ +""" +Health check endpoints for Kubernetes liveness and readiness probes. + +This module provides HTTP endpoints for monitoring the operator's health status: +- /health: General health check (always returns 200 if service is running) +- /ready: Readiness probe (checks if operator is ready to handle requests) +- /live: Liveness probe (checks if operator is alive and not deadlocked) +""" + +import asyncio +import logging +from typing import Optional + +from aiohttp import web + +logger = logging.getLogger(__name__) + + +class HealthCheckServer: + """HTTP server for health check endpoints.""" + + def __init__(self, host: str = "0.0.0.0", port: int = 8080): + """ + Initialize health check server. + + Args: + host: Host to bind to (default: 0.0.0.0 for all interfaces) + port: Port to listen on (default: 8080) + """ + self.host = host + self.port = port + self.app = web.Application() + self._setup_routes() + self.runner: Optional[web.AppRunner] = None + self._is_ready = False + + def _setup_routes(self): + """Configure HTTP routes for health endpoints.""" + self.app.router.add_get("/health", self.health_handler) + self.app.router.add_get("/ready", self.readiness_handler) + self.app.router.add_get("/live", self.liveness_handler) + + async def health_handler(self, request: web.Request) -> web.Response: + """ + General health check endpoint. + + Returns 200 OK if the service is running. + Used for general health monitoring. + """ + return web.json_response({"status": "healthy"}) + + async def readiness_handler(self, request: web.Request) -> web.Response: + """ + Readiness probe endpoint. + + Returns 200 OK if the operator is ready to handle requests. + Returns 503 Service Unavailable if not ready (e.g., during startup). + + Kubernetes uses this to determine when to send traffic to the pod. + """ + if self._is_ready: + return web.json_response({"status": "ready"}) + return web.json_response( + {"status": "not ready", "reason": "operator initializing"}, + status=503, + ) + + async def liveness_handler(self, request: web.Request) -> web.Response: + """ + Liveness probe endpoint. + + Returns 200 OK if the operator is alive and functional. + If this fails, Kubernetes will restart the pod. + + Currently always returns healthy - can be extended to detect deadlocks. + """ + return web.json_response({"status": "alive"}) + + def mark_ready(self): + """Mark the operator as ready to handle requests.""" + self._is_ready = True + logger.info("Operator marked as ready") + + def mark_not_ready(self): + """Mark the operator as not ready (e.g., during shutdown).""" + self._is_ready = False + logger.info("Operator marked as not ready") + + async def start(self): + """Start the health check HTTP server.""" + self.runner = web.AppRunner(self.app) + await self.runner.setup() + site = web.TCPSite(self.runner, self.host, self.port) + await site.start() + logger.info(f"Health check server started on {self.host}:{self.port}") + + async def stop(self): + """Stop the health check HTTP server gracefully.""" + if self.runner: + await self.runner.cleanup() + logger.info("Health check server stopped") + + +async def run_health_server(host: str = "0.0.0.0", port: int = 8080) -> HealthCheckServer: + """ + Create and start a health check server. + + Args: + host: Host to bind to + port: Port to listen on + + Returns: + Running HealthCheckServer instance + """ + server = HealthCheckServer(host, port) + await server.start() + return server diff --git a/input_validation.py b/input_validation.py new file mode 100644 index 0000000..abfa2f1 --- /dev/null +++ b/input_validation.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +""" +Input validation utilities for preventing command injection and other attacks. + +This module provides validation functions for user-controlled inputs like +hostnames, URLs, and other parameters that could be exploited. +""" + +import re +import logging +from typing import Optional +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + + +class ValidationError(Exception): + """Raised when input validation fails""" + + pass + + +def validate_hostname(hostname: str) -> str: + """ + Validate hostname or IP address to prevent command injection. + + Args: + hostname: Hostname or IP address to validate + + Returns: + Validated hostname (same as input if valid) + + Raises: + ValidationError: If hostname is invalid or contains dangerous characters + """ + if not hostname: + raise ValidationError("Hostname cannot be empty") + + if len(hostname) > 253: + raise ValidationError(f"Hostname too long: {len(hostname)} > 253 characters") + + # Allow hostnames, IPv4, and IPv6 + # Hostname pattern: alphanumeric, hyphens, dots + # IPv4: digits and dots + # IPv6: hex digits, colons, brackets (can start with [) + safe_pattern = r"^[\[a-zA-Z0-9]([a-zA-Z0-9\-\.:\[\]])*[a-zA-Z0-9\]]?$" + + if not re.match(safe_pattern, hostname): + raise ValidationError( + f"Hostname contains invalid characters: {hostname}. " + "Only alphanumeric, hyphens, dots, colons, and brackets allowed." + ) + + # Check for dangerous patterns + dangerous_patterns = [ + r";", # Command separator + r"\$", # Variable expansion + r"`", # Command substitution + r"\|", # Pipe + r"&", # Background/AND + r">", # Redirect + r"<", # Redirect + r"\(", # Subshell + r"\)", # Subshell + r"\{", # Brace expansion + r"\}", # Brace expansion + r"\n", # Newline + r"\r", # Carriage return + ] + + for pattern in dangerous_patterns: + if re.search(pattern, hostname): + raise ValidationError( + f"Hostname contains dangerous character: {hostname}" + ) + + logger.debug(f"Validated hostname: {hostname}") + return hostname + + +def validate_git_url(git_url: str) -> str: + """ + Validate Git repository URL to prevent command injection. + + Args: + git_url: Git repository URL to validate + + Returns: + Validated URL (same as input if valid) + + Raises: + ValidationError: If URL is invalid or dangerous + """ + if not git_url: + raise ValidationError("Git URL cannot be empty") + + if len(git_url) > 2048: + raise ValidationError(f"Git URL too long: {len(git_url)} > 2048 characters") + + # Parse URL + try: + parsed = urlparse(git_url) + except Exception as e: + raise ValidationError(f"Invalid URL format: {e}") + + # Allow only safe protocols + allowed_schemes = ["https", "http", "git", "ssh"] + if parsed.scheme and parsed.scheme not in allowed_schemes: + raise ValidationError( + f"Disallowed URL scheme: {parsed.scheme}. " + f"Allowed: {', '.join(allowed_schemes)}" + ) + + # Check for dangerous characters in URL + dangerous_chars = [";", "$", "`", "|", "&", "\n", "\r", "$(", "${"] + for char in dangerous_chars: + if char in git_url: + raise ValidationError(f"Git URL contains dangerous character: {char}") + + logger.debug(f"Validated Git URL: {git_url}") + return git_url + + +def validate_ssh_username(username: str) -> str: + """ + Validate SSH username to prevent command injection. + + Args: + username: SSH username to validate + + Returns: + Validated username (same as input if valid) + + Raises: + ValidationError: If username is invalid + """ + if not username: + raise ValidationError("SSH username cannot be empty") + + if len(username) > 32: + raise ValidationError(f"SSH username too long: {len(username)} > 32 characters") + + # Only allow alphanumeric, underscore, hyphen + if not re.match(r"^[a-zA-Z0-9_\-]+$", username): + raise ValidationError( + f"SSH username contains invalid characters: {username}. " + "Only alphanumeric, underscore, and hyphen allowed." + ) + + logger.debug(f"Validated SSH username: {username}") + return username + + +def validate_path(path: str, max_length: int = 4096) -> str: + """ + Validate file path to prevent directory traversal and injection. + + Args: + path: File path to validate + max_length: Maximum allowed path length + + Returns: + Validated path (same as input if valid) + + Raises: + ValidationError: If path is invalid or dangerous + """ + if not path: + raise ValidationError("Path cannot be empty") + + if len(path) > max_length: + raise ValidationError(f"Path too long: {len(path)} > {max_length} characters") + + # Check for null bytes + if "\x00" in path: + raise ValidationError("Path contains null byte") + + # Check for dangerous patterns + if ".." in path: + logger.warning(f"Path contains parent directory reference: {path}") + # Not necessarily dangerous, but log it + + # Check for command injection characters + dangerous_chars = [";", "$", "`", "|", "&", "\n", "\r"] + for char in dangerous_chars: + if char in path: + raise ValidationError(f"Path contains dangerous character: {char}") + + logger.debug(f"Validated path: {path}") + return path diff --git a/install.sh b/install.sh index 499a60e..1129582 100755 --- a/install.sh +++ b/install.sh @@ -13,12 +13,12 @@ kubectl apply -f crds/ # Сборка и загрузка образа оператора echo "Building operator image..." -docker build -t nixos-operator:latest . +podman build -t nixos-operator:latest . # Если используется kind или minikube, загрузить образ в кластер if command -v kind &> /dev/null; then echo "Loading image into kind cluster..." - kind load docker-image nixos-operator:latest + podman save nixos-operator:latest | kind load image-archive /dev/stdin elif command -v minikube &> /dev/null; then echo "Loading image into minikube cluster..." minikube image load nixos-operator:latest diff --git a/kind-setup.sh b/kind-setup.sh index 3c10259..907f7d9 100755 --- a/kind-setup.sh +++ b/kind-setup.sh @@ -17,9 +17,9 @@ if ! command -v kubectl &> /dev/null; then exit 1 fi -# Проверка наличия Docker -if ! command -v docker &> /dev/null; then - echo "❌ Docker не установлен. Установите Docker: https://docs.docker.com/get-docker/" +# Проверка наличия Podman +if ! command -v podman &> /dev/null; then + echo "❌ Podman не установлен. Установите Podman: https://podman.io/getting-started/installation" exit 1 fi @@ -61,13 +61,13 @@ kubectl create namespace nixos-operator-system --dry-run=client -o yaml | kubect echo "📋 Применение Custom Resource Definitions..." kubectl apply -f crds/ -# Сборка Docker образа оператора -echo "🐳 Сборка Docker образа оператора..." -docker build -t nixos-operator:latest . +# Сборка образа оператора +echo "🐳 Сборка образа оператора..." +podman build -t nixos-operator:latest . # Загрузка образа в kind кластер echo "📤 Загрузка образа в kind кластер..." -kind load docker-image nixos-operator:latest --name "$CLUSTER_NAME" +podman save nixos-operator:latest | kind load image-archive /dev/stdin --name "$CLUSTER_NAME" # Применение deployment echo "🚀 Запуск оператора..." diff --git a/known_hosts_manager.py b/known_hosts_manager.py new file mode 100644 index 0000000..066a6bc --- /dev/null +++ b/known_hosts_manager.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +""" +SSH known_hosts management for secure host verification. + +This module provides utilities for managing SSH host keys and preventing +Man-in-the-Middle attacks by properly verifying host identities. +""" + +import logging +import os +import tempfile +from typing import Optional, Dict, Any +from pathlib import Path + +import config + +logger = logging.getLogger(__name__) + + +class KnownHostsManager: + """Manages SSH known_hosts for host verification""" + + def __init__(self, storage_path: Optional[str] = None): + """ + Initialize known hosts manager. + + Args: + storage_path: Path to store known_hosts file. If None, uses temp directory. + """ + if storage_path: + self.known_hosts_path = Path(storage_path) + else: + # Use persistent temp directory for operator lifetime + temp_dir = Path(config.KNOWN_HOSTS_PATH) + temp_dir.mkdir(parents=True, exist_ok=True) + self.known_hosts_path = temp_dir / "known_hosts" + + # Create file if it doesn't exist + self.known_hosts_path.touch(mode=0o600, exist_ok=True) + logger.info(f"Using known_hosts file: {self.known_hosts_path}") + + def get_known_hosts_path(self) -> str: + """Get path to known_hosts file""" + return str(self.known_hosts_path) + + def add_host_key(self, hostname: str, key_type: str, public_key: str) -> None: + """ + Add a host key to known_hosts. + + Args: + hostname: Hostname or IP address + key_type: Key type (e.g., 'ssh-ed25519', 'ecdsa-sha2-nistp256') + public_key: Base64-encoded public key + """ + entry = f"{hostname} {key_type} {public_key}\n" + + # Check if entry already exists + if self.known_hosts_path.exists(): + with open(self.known_hosts_path, "r") as f: + if entry in f.read(): + logger.debug(f"Host key for {hostname} already in known_hosts") + return + + # Append new entry + with open(self.known_hosts_path, "a") as f: + f.write(entry) + logger.info(f"Added host key for {hostname} to known_hosts") + + def trust_on_first_use(self, hostname: str, port: int = 22) -> bool: + """ + Implement Trust On First Use (TOFU) policy. + + On first connection, accept and store the host key. + On subsequent connections, verify against stored key. + + Args: + hostname: Hostname or IP to connect to + port: SSH port (default 22) + + Returns: + True if this is first connection (key will be added), + False if key already exists (will be verified) + """ + # Check if we have a key for this host + if not self.known_hosts_path.exists(): + logger.info(f"TOFU: First connection to {hostname}, will trust host key") + return True + + with open(self.known_hosts_path, "r") as f: + content = f.read() + # Simple check - does hostname appear in known_hosts? + if hostname in content or f"[{hostname}]:{port}" in content: + logger.debug(f"TOFU: Found existing key for {hostname}") + return False + + logger.info(f"TOFU: First connection to {hostname}, will trust host key") + return True + + def clear_host(self, hostname: str) -> None: + """ + Remove all entries for a specific host. + + Useful when host keys change (e.g., after machine reinstall). + + Args: + hostname: Hostname to remove + """ + if not self.known_hosts_path.exists(): + return + + with open(self.known_hosts_path, "r") as f: + lines = f.readlines() + + # Filter out lines containing this hostname + filtered_lines = [ + line for line in lines if not line.startswith(hostname + " ") and not line.startswith(f"[{hostname}]:") + ] + + with open(self.known_hosts_path, "w") as f: + f.writelines(filtered_lines) + + logger.info(f"Removed host keys for {hostname}") + + +# Global instance for operator lifetime +_known_hosts_manager: Optional[KnownHostsManager] = None + + +def get_known_hosts_manager() -> KnownHostsManager: + """Get or create global known_hosts manager instance""" + global _known_hosts_manager + if _known_hosts_manager is None: + _known_hosts_manager = KnownHostsManager() + return _known_hosts_manager diff --git a/machine_handlers.py b/machine_handlers.py index 35ea162..6618e8c 100644 --- a/machine_handlers.py +++ b/machine_handlers.py @@ -1,270 +1,111 @@ #!/usr/bin/env python3 import logging -import asyncio -import asyncssh -import json -import tempfile import os -from typing import Dict -from datetime import datetime -from clients import get_secret_data -from events import emit_missing_credentials_event +from typing import Dict, Optional, Any + +from ssh_utils import establish_ssh_connection, cleanup_ssh_key +import config logger = logging.getLogger(__name__) async def check_machine_discoverable( - machine_spec: Dict, body=None, machine_name: str = None, namespace: str = None + machine_spec: Dict[str, Any], + body: Optional[Dict[str, Any]] = None, + machine_name: Optional[str] = None, + namespace: Optional[str] = None, ) -> bool: """Check machine availability via SSH with support for key, password, and no authentication""" - try: - ssh_config = { - "host": machine_spec["hostname"], - "username": machine_spec.get("sshUser", "root"), - "known_hosts": None, # Disable known hosts verification - } - - has_credentials = False - ssh_key_temp_file = None - - # Attempt SSH key connection - if "sshKeySecretRef" in machine_spec: - try: - secret_data = await get_secret_data( - machine_spec["sshKeySecretRef"]["name"], - machine_spec["sshKeySecretRef"].get("namespace", "default"), - ) - if "ssh-privatekey" in secret_data and secret_data["ssh-privatekey"]: - # Create temporary file for SSH key - with tempfile.NamedTemporaryFile( - mode="w", delete=False, suffix="_ssh_key" - ) as temp_file: - temp_file.write(secret_data["ssh-privatekey"]) - ssh_key_temp_file = temp_file.name - - # Set correct permissions for SSH key - os.chmod(ssh_key_temp_file, 0o600) - - ssh_config["client_keys"] = [ssh_key_temp_file] - has_credentials = True - logger.info("Using SSH key for authentication") - else: - # Secret exists but doesn't contain SSH key - if body: - emit_missing_credentials_event( - body, - "MissingSSHKey", - f"Secret {machine_spec['sshKeySecretRef']['name']} exists but doesn't contain 'ssh-privatekey'", - ) - logger.warning( - f"Secret {machine_spec['sshKeySecretRef']['name']} exists but doesn't contain 'ssh-privatekey'" - ) - except Exception as e: - # Secret not found or unavailable - if body: - emit_missing_credentials_event( - body, - "SecretNotFound", - f"Failed to get SSH key from secret {machine_spec['sshKeySecretRef']['name']}", - ) - logger.warning( - f"Failed to get SSH key from secret {machine_spec['sshKeySecretRef']['name']}: {e}" - ) - - # Attempt password connection (if key didn't work or not specified) - if not has_credentials and "sshPasswordSecretRef" in machine_spec: - try: - secret_data = await get_secret_data( - machine_spec["sshPasswordSecretRef"]["name"], - machine_spec["sshPasswordSecretRef"].get("namespace", "default"), - ) + conn, ssh_key_temp_file = await establish_ssh_connection( + machine_spec, body, machine_name, namespace + ) - # Determine password key (default 'password') - password_key = machine_spec["sshPasswordSecretRef"].get( - "key", "password" - ) - - if password_key in secret_data and secret_data[password_key]: - ssh_config["password"] = secret_data[password_key] - has_credentials = True - logger.info("Using password for authentication") - else: - # Secret exists but doesn't contain password - if body: - emit_missing_credentials_event( - body, - "MissingPassword", - f"Secret {machine_spec['sshPasswordSecretRef']['name']} exists but doesn't contain '{password_key}'", - ) - logger.warning( - f"Secret {machine_spec['sshPasswordSecretRef']['name']} exists but doesn't contain '{password_key}'" - ) - except Exception as e: - # Secret not found or unavailable - if body: - emit_missing_credentials_event( - body, - "SecretNotFound", - f"Failed to get password from secret {machine_spec['sshPasswordSecretRef']['name']}", - ) - logger.warning( - f"Failed to get password from secret {machine_spec['sshPasswordSecretRef']['name']}: {e}" - ) - - # If no credentials provided, try connection without authentication - if not has_credentials: - logger.info( - "No SSH key or password provided, attempting connection without authentication" - ) - # Continue without additional authentication parameters - - # Attempt connection - try: - async with asyncssh.connect(**ssh_config) as conn: - # Simple command to check availability - result = await conn.run('echo "machine_available"', check=True) - return result.stdout.strip() == "machine_available" - finally: - # Delete temporary SSH key file if it was created - if ssh_key_temp_file and os.path.exists(ssh_key_temp_file): - try: - os.unlink(ssh_key_temp_file) - logger.debug(f"Deleted temporary SSH key file: {ssh_key_temp_file}") - except Exception as e: - logger.warning( - f"Failed to delete temporary SSH key file {ssh_key_temp_file}: {e}" - ) + if not conn: + return False + try: + # Simple command to check availability + result = await conn.run('echo "machine_available"', check=True) + return result.stdout.strip() == "machine_available" except Exception as e: logger.warning( - f"Machine {machine_spec.get('hostname')} is not discoverable: {e}" + f"Machine {machine_spec.get('hostname')} availability check failed: {e}" ) return False + finally: + conn.close() + await conn.wait_closed() + cleanup_ssh_key(ssh_key_temp_file) async def scan_machine_hardware( - machine_spec: Dict, body=None, machine_name: str = None, namespace: str = None -) -> Dict: + machine_spec: Dict[str, Any], + body: Optional[Dict[str, Any]] = None, + machine_name: Optional[str] = None, + namespace: Optional[str] = None, +) -> Dict[str, Any]: """Scan machine hardware and return facts""" - try: - ssh_config = { - "host": machine_spec["hostname"], - "username": machine_spec.get("sshUser", "root"), - "known_hosts": None, # Disable known hosts verification - } + conn, ssh_key_temp_file = await establish_ssh_connection( + machine_spec, body, machine_name, namespace + ) - has_credentials = False - ssh_key_temp_file = None - - # Attempt SSH key connection - if "sshKeySecretRef" in machine_spec: - try: - secret_data = await get_secret_data( - machine_spec["sshKeySecretRef"]["name"], - machine_spec["sshKeySecretRef"].get("namespace", "default"), - ) - if "ssh-privatekey" in secret_data and secret_data["ssh-privatekey"]: - # Create temporary file for SSH key - with tempfile.NamedTemporaryFile( - mode="w", delete=False, suffix="_ssh_key" - ) as temp_file: - temp_file.write(secret_data["ssh-privatekey"]) - ssh_key_temp_file = temp_file.name - - # Set correct permissions for SSH key - os.chmod(ssh_key_temp_file, 0o600) - - ssh_config["client_keys"] = [ssh_key_temp_file] - has_credentials = True - logger.info("Using SSH key for hardware scan") - except Exception as e: - logger.warning(f"Failed to get SSH key for hardware scan: {e}") - - # Attempt password connection (if key didn't work or not specified) - if not has_credentials and "sshPasswordSecretRef" in machine_spec: - try: - secret_data = await get_secret_data( - machine_spec["sshPasswordSecretRef"]["name"], - machine_spec["sshPasswordSecretRef"].get("namespace", "default"), - ) - - # Determine password key (default 'password') - password_key = machine_spec["sshPasswordSecretRef"].get( - "key", "password" - ) - - if password_key in secret_data and secret_data[password_key]: - ssh_config["password"] = secret_data[password_key] - has_credentials = True - logger.info("Using password for hardware scan") - except Exception as e: - logger.warning(f"Failed to get password for hardware scan: {e}") - - # If no credentials provided, try connection without authentication - if not has_credentials: - logger.info( - "No SSH key or password provided, attempting hardware scan without authentication" - ) + if not conn: + logger.warning( + f"Failed to connect to machine {machine_spec.get('hostname')} for hardware scan" + ) + return {} - # Connection and scan execution - try: - async with asyncssh.connect(**ssh_config) as conn: - # Transfer scan script via SCP - scanner_path = os.path.join( - os.path.dirname(__file__), "scripts", "hardware_scanner.sh" - ) + try: + # Transfer scan script via SCP + scanner_path = os.path.join( + os.path.dirname(__file__), "scripts", "hardware_scanner.sh" + ) - if not os.path.exists(scanner_path): - logger.error(f"Hardware scanner script not found at {scanner_path}") - return {} + if not os.path.exists(scanner_path): + logger.error(f"Hardware scanner script not found at {scanner_path}") + return {} - # Read script content - with open(scanner_path, "r") as f: - scanner_content = f.read() + # Read script content + with open(scanner_path, "r") as f: + scanner_content = f.read() - # Create temporary file on remote machine - remote_script_path = "/tmp/hardware_scanner.sh" + # Create temporary file on remote machine + remote_script_path = config.REMOTE_HARDWARE_SCRIPT_PATH - # Transfer script via SCP - async with conn.start_sftp_client() as sftp: - async with sftp.open(remote_script_path, "w") as remote_file: - await remote_file.write(scanner_content) + # Transfer script via SCP + async with conn.start_sftp_client() as sftp: + async with sftp.open(remote_script_path, "w") as remote_file: + await remote_file.write(scanner_content) - # Make script executable and run it - await conn.run(f"chmod +x {remote_script_path}", check=True) - result = await conn.run(f"{remote_script_path}", check=True) + # Make script executable and run it + await conn.run(f"chmod +x {remote_script_path}", check=True) + result = await conn.run(f"{remote_script_path}", check=True) - # Get raw scanner output - facts_output = result.stdout.strip() - if not facts_output: - logger.warning("Hardware scanner returned empty output") - return {} + # Get raw scanner output + facts_output = result.stdout.strip() + if not facts_output: + logger.warning("Hardware scanner returned empty output") + return {} - # Parse result locally - from scripts.facts_parser import parse_facts + # Parse result locally + from scripts.facts_parser import parse_facts - # Split output into lines and parse - lines = facts_output.split("\n") - hardware_facts = parse_facts(lines) + # Split output into lines and parse + lines = facts_output.split("\n") + hardware_facts = parse_facts(lines) - logger.info( - f"Successfully scanned hardware for machine {machine_spec['hostname']}" - ) - return hardware_facts - finally: - # Delete temporary SSH key file if it was created - if ssh_key_temp_file and os.path.exists(ssh_key_temp_file): - try: - os.unlink(ssh_key_temp_file) - logger.debug(f"Deleted temporary SSH key file: {ssh_key_temp_file}") - except Exception as e: - logger.warning( - f"Failed to delete temporary SSH key file {ssh_key_temp_file}: {e}" - ) + logger.info( + f"Successfully scanned hardware for machine {machine_spec['hostname']}" + ) + return hardware_facts except Exception as e: logger.warning( f"Failed to scan hardware for machine {machine_spec.get('hostname')}: {e}" ) return {} + finally: + conn.close() + await conn.wait_closed() + cleanup_ssh_key(ssh_key_temp_file) diff --git a/main.py b/main.py index cfdd83e..06435e2 100644 --- a/main.py +++ b/main.py @@ -2,17 +2,29 @@ import kopf import logging +import os +import signal +import asyncio from machine_handlers import check_machine_discoverable, scan_machine_hardware from nixosconfiguration_handlers import reconcile_nixos_configuration from clients import update_machine_status, get_machine -import os +from metrics import init_metrics +from prometheus_client import start_http_server +from health import run_health_server, HealthCheckServer +import config # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -print("starting") +logger.info("NixOS Infrastructure Operator starting") + +# Global flag for graceful shutdown +_shutdown_event = asyncio.Event() + +# Global health check server instance +_health_server: HealthCheckServer = None # --- Add Nix path to PATH --- nix_bin_path = "/nix/var/nix/profiles/default/bin" @@ -25,48 +37,46 @@ # Machine handlers @kopf.on.create("nio.homystack.com", "v1alpha1", "machines") async def on_machine_create(body, spec, name, namespace, **kwargs): - """Обработчик создания Machine""" + """Handler for Machine resource creation""" logger.info(f"Creating Machine: {name}") - # Проверка доступности машины с передачей body для событий + # Check machine availability with body for events is_discoverable = await check_machine_discoverable(spec, body, name, namespace) - # Установка начального статуса + # Set initial status await update_machine_status( name, namespace, {"discoverable": is_discoverable, "hasConfiguration": False} ) -@kopf.timer("nio.homystack.com", "v1alpha1", "machines", interval=60.0) +@kopf.timer("nio.homystack.com", "v1alpha1", "machines", interval=config.MACHINE_DISCOVERY_INTERVAL) async def check_machine_discoverability(body, spec, name, namespace, **kwargs): - """Периодическая проверка доступности машин""" + """Periodic machine availability check""" logger.debug(f"Checking discoverability for machine: {name}") - # Проверка доступности машины с передачей body для событий + # Check machine availability with body for events is_discoverable = await check_machine_discoverable(spec, body, name, namespace) - # Обновление статуса + # Update status await update_machine_status(name, namespace, {"discoverable": is_discoverable}) -@kopf.timer("nio.homystack.com", "v1alpha1", "machines", interval=300.0) # Каждые 5 минут +@kopf.timer("nio.homystack.com", "v1alpha1", "machines", interval=config.HARDWARE_SCAN_INTERVAL) async def scan_machine_hardware_periodically(body, spec, name, namespace, **kwargs): - """Периодическое сканирование железа машин""" + """Periodic hardware scanning for machines""" logger.debug(f"Scanning hardware for machine: {name}") - # Проверяем доступность машины перед сканированием + # Check machine availability before scanning is_discoverable = await check_machine_discoverable(spec, body, name, namespace) if not is_discoverable: logger.warning(f"Machine {name} is not discoverable, skipping hardware scan") return - # Сканируем железо + # Scan hardware hardware_facts = await scan_machine_hardware(spec, body, name, namespace) - # Обновляем статус с фактами о железе - from datetime import datetime - + # Update status with hardware facts await update_machine_status( name, namespace, @@ -81,16 +91,79 @@ async def scan_machine_hardware_periodically(body, spec, name, namespace, **kwar @kopf.on.update("nio.homystack.com", "v1alpha1", "nixosconfigurations") @kopf.on.resume("nio.homystack.com", "v1alpha1", "nixosconfigurations") @kopf.on.delete("nio.homystack.com", "v1alpha1", "nixosconfigurations") -@kopf.on.timer("nio.homystack.com", "v1alpha1", "nixosconfigurations", interval=120) +@kopf.on.timer("nio.homystack.com", "v1alpha1", "nixosconfigurations", interval=config.CONFIG_RECONCILE_INTERVAL) async def unified_nixos_configuration_handler(body, spec, name, namespace, **kwargs): - """Унифицированный обработчик для всех операций с NixosConfiguration""" + """Unified handler for all NixosConfiguration operations""" await reconcile_nixos_configuration(body, spec, name, namespace, **kwargs) @kopf.on.startup() -def configure(settings: kopf.OperatorSettings, **_): +async def configure(settings: kopf.OperatorSettings, **_): + global _health_server + settings.posting.level = logging.WARNING + # Initialize Prometheus metrics + init_metrics() + + # Start Prometheus metrics server + start_http_server(config.METRICS_PORT) + logger.info(f"Prometheus metrics server started on port {config.METRICS_PORT}") + + # Start health check server + _health_server = await run_health_server( + host="0.0.0.0", + port=config.HEALTH_CHECK_PORT + ) + # Mark as ready after initialization + _health_server.mark_ready() + logger.info(f"Health check server started on port {config.HEALTH_CHECK_PORT}") + + # Log configuration summary + logger.info(config.get_config_summary()) + + +def handle_shutdown_signal(signum, frame): + """Handle shutdown signals for graceful termination""" + signal_name = signal.Signals(signum).name + logger.info(f"Received {signal_name} signal, initiating graceful shutdown...") + _shutdown_event.set() + + +@kopf.on.cleanup() +async def cleanup_handler(**kwargs): + """Cleanup handler called on operator shutdown""" + global _health_server + + logger.info("Operator cleanup: draining active reconciliations...") + + # Mark as not ready to stop receiving traffic + if _health_server: + _health_server.mark_not_ready() + + # Give active reconciliations time to complete + await asyncio.sleep(5) + + # Stop health check server + if _health_server: + await _health_server.stop() + + logger.info("Operator cleanup complete") + if __name__ == "__main__": - kopf.run() + # Register signal handlers for graceful shutdown + signal.signal(signal.SIGTERM, handle_shutdown_signal) + signal.signal(signal.SIGINT, handle_shutdown_signal) + + logger.info("Signal handlers registered (SIGTERM, SIGINT)") + + try: + kopf.run() + except KeyboardInterrupt: + logger.info("Operator stopped by user") + except Exception as e: + logger.error(f"Operator crashed: {e}", exc_info=True) + raise + finally: + logger.info("NixOS Infrastructure Operator shutdown complete") diff --git a/metrics.py b/metrics.py new file mode 100644 index 0000000..67c5fb2 --- /dev/null +++ b/metrics.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +""" +Prometheus metrics for the NixOS Infrastructure Operator. + +Provides observability into operator health, performance, and resource states. +""" + +import logging +from prometheus_client import Counter, Gauge, Histogram, Info + +logger = logging.getLogger(__name__) + +# Info metrics +operator_info = Info("nio_operator", "NixOS Infrastructure Operator information") + +# Machine metrics +machines_total = Gauge( + "nio_machines_total", + "Total number of managed machines", + ["namespace"], +) + +machines_discoverable = Gauge( + "nio_machines_discoverable", + "Number of discoverable machines", + ["namespace"], +) + +machines_with_configuration = Gauge( + "nio_machines_with_configuration", + "Number of machines with applied configuration", + ["namespace"], +) + +# Configuration metrics +configurations_total = Gauge( + "nio_configurations_total", + "Total number of NixOS configurations", + ["namespace"], +) + +configurations_applied = Counter( + "nio_configurations_applied_total", + "Total number of successful configuration applications", + ["namespace", "machine"], +) + +configurations_failed = Counter( + "nio_configurations_failed_total", + "Total number of failed configuration applications", + ["namespace", "machine", "reason"], +) + +# Reconciliation metrics +reconcile_duration = Histogram( + "nio_reconcile_duration_seconds", + "Time spent reconciling configurations", + ["namespace", "configuration"], + buckets=(1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600), # Up to 1 hour +) + +reconcile_errors = Counter( + "nio_reconcile_errors_total", + "Total number of reconciliation errors", + ["namespace", "configuration", "error_type"], +) + +# SSH connection metrics +ssh_connections_total = Counter( + "nio_ssh_connections_total", + "Total number of SSH connection attempts", + ["namespace", "machine", "result"], +) + +ssh_connection_duration = Histogram( + "nio_ssh_connection_duration_seconds", + "Time to establish SSH connections", + ["namespace", "machine"], + buckets=(0.1, 0.5, 1, 2, 5, 10, 30), +) + +# Git operations metrics +git_clones_total = Counter( + "nio_git_clones_total", + "Total number of Git clone operations", + ["namespace", "repository", "result"], +) + +git_clone_duration = Histogram( + "nio_git_clone_duration_seconds", + "Time to clone Git repositories", + ["namespace", "repository"], + buckets=(1, 5, 10, 30, 60, 120, 300), +) + +# NixOS operations metrics +nixos_builds_total = Counter( + "nio_nixos_builds_total", + "Total number of NixOS builds", + ["namespace", "machine", "build_type", "result"], +) + +nixos_build_duration = Histogram( + "nio_nixos_build_duration_seconds", + "Time to build and apply NixOS configurations", + ["namespace", "machine", "build_type"], + buckets=(60, 300, 600, 1200, 1800, 3600, 7200), # Up to 2 hours +) + +# Retry metrics +retries_total = Counter( + "nio_retries_total", + "Total number of operation retries", + ["operation", "attempt"], +) + +retries_exhausted = Counter( + "nio_retries_exhausted_total", + "Total number of operations that exhausted all retries", + ["operation"], +) + +# Error metrics +errors_total = Counter( + "nio_errors_total", + "Total number of errors by type", + ["error_type", "component"], +) + +# Validation metrics +validation_errors = Counter( + "nio_validation_errors_total", + "Total number of input validation errors", + ["validation_type", "field"], +) + + +def init_metrics(): + """Initialize metrics with operator information""" + operator_info.info( + { + "version": "0.2.0", + "name": "nixos-infrastructure-operator", + "component": "operator", + } + ) + logger.info("Prometheus metrics initialized") + + +# Helper functions for common metric operations +def record_reconcile_success(namespace: str, configuration: str, duration: float): + """Record successful reconciliation""" + reconcile_duration.labels(namespace=namespace, configuration=configuration).observe(duration) + + +def record_reconcile_error(namespace: str, configuration: str, error_type: str): + """Record reconciliation error""" + reconcile_errors.labels( + namespace=namespace, configuration=configuration, error_type=error_type + ).inc() + + +def record_ssh_connection(namespace: str, machine: str, success: bool, duration: float): + """Record SSH connection attempt""" + result = "success" if success else "failure" + ssh_connections_total.labels(namespace=namespace, machine=machine, result=result).inc() + if success: + ssh_connection_duration.labels(namespace=namespace, machine=machine).observe(duration) + + +def record_git_clone(namespace: str, repository: str, success: bool, duration: float): + """Record Git clone operation""" + result = "success" if success else "failure" + git_clones_total.labels(namespace=namespace, repository=repository, result=result).inc() + if success: + git_clone_duration.labels(namespace=namespace, repository=repository).observe(duration) + + +def record_nixos_build( + namespace: str, machine: str, build_type: str, success: bool, duration: float +): + """Record NixOS build operation""" + result = "success" if success else "failure" + nixos_builds_total.labels( + namespace=namespace, machine=machine, build_type=build_type, result=result + ).inc() + if success: + nixos_build_duration.labels( + namespace=namespace, machine=machine, build_type=build_type + ).observe(duration) diff --git a/monitoring/grafana/nio-operator-dashboard.json b/monitoring/grafana/nio-operator-dashboard.json new file mode 100644 index 0000000..99b5611 --- /dev/null +++ b/monitoring/grafana/nio-operator-dashboard.json @@ -0,0 +1,1770 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of managed machines across all namespaces", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(nio_machines_total)", + "legendFormat": "Total Machines", + "range": true, + "refId": "A" + } + ], + "title": "Total Machines", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of machines currently discoverable via SSH", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(nio_machines_discoverable)", + "legendFormat": "Discoverable", + "range": true, + "refId": "A" + } + ], + "title": "Discoverable Machines", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of NixOS configurations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(nio_configurations_total)", + "legendFormat": "Total Configs", + "range": true, + "refId": "A" + } + ], + "title": "Total Configurations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Machines with successfully applied configuration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(nio_machines_with_configuration)", + "legendFormat": "Configured", + "range": true, + "refId": "A" + } + ], + "title": "Machines with Config", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 101, + "panels": [], + "title": "Reconciliation Performance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of successful configuration applications", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_configurations_applied_total[5m])", + "legendFormat": "{{namespace}}/{{machine}}", + "range": true, + "refId": "A" + } + ], + "title": "Configuration Application Rate (Success)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of failed configuration applications by reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_configurations_failed_total[5m])", + "legendFormat": "{{namespace}}/{{machine}} - {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Configuration Application Rate (Failed)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "P95 reconciliation duration by configuration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "red", + "value": 600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(nio_reconcile_duration_seconds_bucket[5m])) by (le, namespace, configuration))", + "legendFormat": "{{namespace}}/{{configuration}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconciliation Duration (P95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Reconciliation errors by type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_reconcile_errors_total[5m])", + "legendFormat": "{{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconciliation Errors by Type", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 102, + "panels": [], + "title": "SSH & Git Operations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "SSH connection success vs failure rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*success.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*failure.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 9, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_ssh_connections_total[5m])", + "legendFormat": "{{result}} - {{machine}}", + "range": true, + "refId": "A" + } + ], + "title": "SSH Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "P95 SSH connection establishment time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(nio_ssh_connection_duration_seconds_bucket[5m])) by (le, machine))", + "legendFormat": "{{machine}}", + "range": true, + "refId": "A" + } + ], + "title": "SSH Connection Duration (P95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Git clone operations success vs failure rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*success.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*failure.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_git_clones_total[5m])", + "legendFormat": "{{result}} - {{repository}}", + "range": true, + "refId": "A" + } + ], + "title": "Git Clone Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "P95 Git clone operation duration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(nio_git_clone_duration_seconds_bucket[5m])) by (le, repository))", + "legendFormat": "{{repository}}", + "range": true, + "refId": "A" + } + ], + "title": "Git Clone Duration (P95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 103, + "panels": [], + "title": "NixOS Builds", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "NixOS build operations by result", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*success.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*failure.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_nixos_builds_total[5m])", + "legendFormat": "{{result}} - {{build_type}} - {{machine}}", + "range": true, + "refId": "A" + } + ], + "title": "NixOS Build Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "P95 NixOS build duration by type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1200 + }, + { + "color": "red", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(nio_nixos_build_duration_seconds_bucket[5m])) by (le, build_type, machine))", + "legendFormat": "{{build_type}} - {{machine}}", + "range": true, + "refId": "A" + } + ], + "title": "NixOS Build Duration (P95)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 104, + "panels": [], + "title": "Errors & Retries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Error rate by type and component", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 15, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_errors_total[5m])", + "legendFormat": "{{component}} - {{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Validation errors by type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 16, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_validation_errors_total[5m])", + "legendFormat": "{{validation_type}} - {{field}}", + "range": true, + "refId": "A" + } + ], + "title": "Validation Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Retry attempts by operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 17, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_retries_total[5m])", + "legendFormat": "{{operation}} - attempt {{attempt}}", + "range": true, + "refId": "A" + } + ], + "title": "Retry Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Operations that exhausted all retry attempts", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 18, + "options": { + "legend": { + "calcs": ["sum"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nio_retries_exhausted_total[5m])", + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Exhausted Retries Rate", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["nixos", "operator", "infrastructure"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "NixOS Infrastructure Operator", + "uid": "nio-operator", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/prometheus-rules.yaml b/monitoring/prometheus-rules.yaml new file mode 100644 index 0000000..26dcb71 --- /dev/null +++ b/monitoring/prometheus-rules.yaml @@ -0,0 +1,258 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: nixos-operator-alerts + namespace: nixos-operator-system + labels: + app: nixos-operator + prometheus: kube-prometheus +spec: + groups: + - name: nixos-operator.rules + interval: 30s + rules: + # Operator health alerts + - alert: NixOSOperatorDown + expr: up{job="nixos-operator-metrics"} == 0 + for: 5m + labels: + severity: critical + component: operator + annotations: + summary: "NixOS Operator is down" + description: "NixOS Infrastructure Operator has been down for more than 5 minutes. No reconciliation is happening." + runbook_url: "https://github.com/homystack/NIO/blob/main/docs/runbooks/operator-down.md" + + - alert: NixOSOperatorNotReady + expr: | + nio_operator_info{version!=""} == 0 + or absent(nio_operator_info) + for: 10m + labels: + severity: warning + component: operator + annotations: + summary: "NixOS Operator is not ready" + description: "NixOS Infrastructure Operator has not reported ready status for 10 minutes." + + # Reconciliation alerts + - alert: HighReconciliationFailureRate + expr: | + ( + sum(rate(nio_configurations_failed_total[5m])) + / + (sum(rate(nio_configurations_applied_total[5m])) + sum(rate(nio_configurations_failed_total[5m]))) + ) > 0.5 + for: 15m + labels: + severity: warning + component: reconciliation + annotations: + summary: "High configuration reconciliation failure rate" + description: "More than 50% of configuration reconciliations are failing in the last 15 minutes. Current rate: {{ $value | humanizePercentage }}." + + - alert: ReconciliationStuck + expr: | + (time() - max(nio_reconcile_duration_seconds_sum) by (namespace, configuration)) > 7200 + for: 15m + labels: + severity: critical + component: reconciliation + annotations: + summary: "Reconciliation appears stuck" + description: "Configuration {{ $labels.configuration }} in namespace {{ $labels.namespace }} hasn't completed reconciliation in over 2 hours." + + - alert: SlowReconciliation + expr: | + histogram_quantile(0.95, sum(rate(nio_reconcile_duration_seconds_bucket[10m])) by (le, namespace, configuration)) > 1800 + for: 30m + labels: + severity: warning + component: reconciliation + annotations: + summary: "Slow configuration reconciliation" + description: "P95 reconciliation duration for {{ $labels.configuration }} in {{ $labels.namespace }} is above 30 minutes: {{ $value | humanizeDuration }}." + + # SSH connection alerts + - alert: HighSSHConnectionFailureRate + expr: | + ( + sum(rate(nio_ssh_connections_total{result="failure"}[5m])) + / + sum(rate(nio_ssh_connections_total[5m])) + ) > 0.3 + for: 10m + labels: + severity: warning + component: ssh + annotations: + summary: "High SSH connection failure rate" + description: "More than 30% of SSH connection attempts are failing. Current rate: {{ $value | humanizePercentage }}." + + - alert: SSHConnectionsCompletelyFailing + expr: | + ( + sum(rate(nio_ssh_connections_total{result="success"}[10m])) + == + 0 + ) + and + ( + sum(rate(nio_ssh_connections_total{result="failure"}[10m])) + > + 0 + ) + for: 15m + labels: + severity: critical + component: ssh + annotations: + summary: "All SSH connections failing" + description: "All SSH connection attempts have been failing for the past 15 minutes. Machines are unreachable." + + - alert: SlowSSHConnections + expr: | + histogram_quantile(0.95, sum(rate(nio_ssh_connection_duration_seconds_bucket[10m])) by (le, machine)) > 10 + for: 20m + labels: + severity: warning + component: ssh + annotations: + summary: "Slow SSH connections" + description: "P95 SSH connection time to {{ $labels.machine }} is above 10 seconds: {{ $value | humanizeDuration }}." + + # Git operation alerts + - alert: HighGitCloneFailureRate + expr: | + ( + sum(rate(nio_git_clones_total{result="failure"}[5m])) + / + sum(rate(nio_git_clones_total[5m])) + ) > 0.3 + for: 15m + labels: + severity: warning + component: git + annotations: + summary: "High Git clone failure rate" + description: "More than 30% of Git clone operations are failing. Current rate: {{ $value | humanizePercentage }}. Check repository access and network connectivity." + + # NixOS build alerts + - alert: HighNixOSBuildFailureRate + expr: | + ( + sum(rate(nio_nixos_builds_total{result="failure"}[10m])) + / + sum(rate(nio_nixos_builds_total[10m])) + ) > 0.4 + for: 20m + labels: + severity: warning + component: nixos-build + annotations: + summary: "High NixOS build failure rate" + description: "More than 40% of NixOS builds are failing. Current rate: {{ $value | humanizePercentage }}. Check configurations and build logs." + + - alert: SlowNixOSBuilds + expr: | + histogram_quantile(0.95, sum(rate(nio_nixos_build_duration_seconds_bucket[15m])) by (le, machine, build_type)) > 5400 + for: 30m + labels: + severity: info + component: nixos-build + annotations: + summary: "Slow NixOS builds" + description: "P95 NixOS build time for {{ $labels.build_type }} on {{ $labels.machine }} is above 90 minutes: {{ $value | humanizeDuration }}." + + # Machine status alerts + - alert: MachinesNotDiscoverable + expr: | + (nio_machines_total - nio_machines_discoverable) > 0 + for: 30m + labels: + severity: warning + component: machines + annotations: + summary: "Machines not discoverable" + description: "{{ $value }} machine(s) in namespace {{ $labels.namespace }} are not discoverable via SSH. Check network connectivity and SSH configuration." + + - alert: NoMachinesWithConfiguration + expr: | + nio_machines_total > 0 and nio_machines_with_configuration == 0 + for: 1h + labels: + severity: warning + component: machines + annotations: + summary: "No machines have applied configuration" + description: "There are {{ $value }} managed machines but none have successfully applied configuration in namespace {{ $labels.namespace }}." + + # Error rate alerts + - alert: HighErrorRate + expr: | + sum(rate(nio_errors_total[5m])) by (component) > 1 + for: 10m + labels: + severity: warning + component: errors + annotations: + summary: "High error rate in component" + description: "Component {{ $labels.component }} is experiencing high error rate: {{ $value | humanize }} errors/second." + + - alert: HighValidationErrorRate + expr: | + sum(rate(nio_validation_errors_total[5m])) > 0.5 + for: 15m + labels: + severity: info + component: validation + annotations: + summary: "High input validation error rate" + description: "High rate of input validation errors detected: {{ $value | humanize }} errors/second. Check CRD specifications and user inputs." + + # Retry alerts + - alert: HighRetryExhaustionRate + expr: | + sum(rate(nio_retries_exhausted_total[10m])) by (operation) > 0.1 + for: 15m + labels: + severity: warning + component: retries + annotations: + summary: "High retry exhaustion rate" + description: "Operation {{ $labels.operation }} is exhausting all retry attempts at a high rate: {{ $value | humanize }} exhaustions/second. Indicates persistent failures." + + # Reconciliation error alerts by type + - alert: ReconciliationErrorsIncreasing + expr: | + sum(rate(nio_reconcile_errors_total[15m])) by (namespace, configuration, error_type) > 0.05 + for: 20m + labels: + severity: info + component: reconciliation + annotations: + summary: "Increasing reconciliation errors" + description: "Reconciliation errors of type {{ $labels.error_type }} for {{ $labels.configuration }} in {{ $labels.namespace }} are increasing: {{ $value | humanize }} errors/second." + + # Health check alerts (requires health endpoint scraping) + - alert: OperatorHealthCheckFailing + expr: | + probe_success{job="nixos-operator-health"} == 0 + for: 5m + labels: + severity: critical + component: health + annotations: + summary: "Operator health check failing" + description: "Health check endpoint is failing. Operator may be unhealthy or unreachable." + + - alert: OperatorReadinessCheckFailing + expr: | + probe_success{job="nixos-operator-readiness"} == 0 + for: 10m + labels: + severity: warning + component: health + annotations: + summary: "Operator readiness check failing" + description: "Readiness check endpoint is failing. Operator may not be ready to process requests." diff --git a/monitoring/service-monitor.yaml b/monitoring/service-monitor.yaml new file mode 100644 index 0000000..390172f --- /dev/null +++ b/monitoring/service-monitor.yaml @@ -0,0 +1,28 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: nixos-operator + namespace: nixos-operator-system + labels: + app: nixos-operator + prometheus: kube-prometheus +spec: + selector: + matchLabels: + app: nixos-operator + endpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s + path: /metrics + scheme: http + relabelings: + - sourceLabels: [__meta_kubernetes_pod_name] + targetLabel: pod + - sourceLabels: [__meta_kubernetes_namespace] + targetLabel: namespace + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + namespaceSelector: + matchNames: + - nixos-operator-system diff --git a/nixosconfiguration_handlers.py b/nixosconfiguration_handlers.py index 75a2695..00e3f58 100644 --- a/nixosconfiguration_handlers.py +++ b/nixosconfiguration_handlers.py @@ -3,14 +3,16 @@ import logging import shutil import asyncio -from datetime import datetime import tempfile import kopf import os import json import hashlib -from typing import Dict, Optional import subprocess +from datetime import datetime +from typing import Dict, Optional, Any + +import config from machine_handlers import check_machine_discoverable from clients import ( @@ -32,9 +34,9 @@ async def inject_additional_files( repo_path: str, - config_spec: dict, + config_spec: Dict[str, Any], namespace: str, - machine_spec: Optional[dict] = None, + machine_spec: Optional[Dict[str, Any]] = None, ) -> str: """Inject additionalFiles into configurationSubdir and return directory hash""" if not config_spec.get("additionalFiles"): @@ -43,7 +45,7 @@ async def inject_additional_files( config_subdir = config_spec.get("configurationSubdir", "") base_path = os.path.join(repo_path, config_subdir) if config_subdir else repo_path - injected_files = [] # 👈 Store paths of injected files + injected_files = [] # Store paths of injected files for file_spec in config_spec["additionalFiles"]: file_path = os.path.join(base_path, file_spec["path"]) @@ -60,7 +62,7 @@ async def inject_additional_files( with open(file_path, "w") as f: f.write(content) logger.info(f"Injected inline file: {file_spec['path']}") - injected_files.append(file_path) # 👈 Add to list + injected_files.append(file_path) elif value_type == "SecretRef": secret_ref = file_spec.get("secretRef", {}) @@ -80,7 +82,7 @@ async def inject_additional_files( logger.info( f"Injected secret file: {file_spec['path']} from secret {secret_name}" ) - injected_files.append(file_path) # 👈 Add to list + injected_files.append(file_path) else: logger.warning( f"Empty secret {secret_name} for file {file_spec['path']}" @@ -101,9 +103,9 @@ async def inject_additional_files( with open(file_path, "w") as f: f.write(content) logger.info(f"Generated NixosFacter file: {file_spec['path']}") - injected_files.append(file_path) # 👈 Add to list + injected_files.append(file_path) - # 👇 ADD FILES TO GIT INDEX WITHOUT COMMIT + # Add files to git index without commit if injected_files: try: # Add each file to git index with --intend-to-add @@ -132,7 +134,7 @@ async def inject_additional_files( return calculate_directory_hash(base_path) -def generate_nixos_facts(machine_spec: dict) -> dict: +def generate_nixos_facts(machine_spec: Dict[str, Any]) -> Dict[str, Any]: """Generate NixOS facts for machine""" facts = { "machine-id": machine_spec.get("hostname", "unknown"), @@ -148,7 +150,7 @@ def generate_nixos_facts(machine_spec: dict) -> dict: def get_additional_files_hash( - config_spec: dict, namespace: str, machine_spec: Optional[dict] = None + config_spec: Dict[str, Any], namespace: str, machine_spec: Optional[dict] = None ) -> str: """Calculate hash from additionalFiles specification""" if not config_spec.get("additionalFiles"): @@ -170,14 +172,16 @@ def get_additional_files_hash( if machine_spec: file_info["nixosFacter"] = generate_nixos_facts(machine_spec) - content_str = json.dumps(file_info, sort_keys=True) + files_content.append(file_info) + + content_str = json.dumps(files_content, sort_keys=True) return hashlib.sha256(content_str.encode("utf-8")).hexdigest() # ... async def apply_nixos_configuration( - machine_spec: dict, - config_spec: dict, + machine_spec: Dict[str, Any], + config_spec: Dict[str, Any], repo_path: str, commit_hash: str, is_remove: bool, @@ -212,16 +216,21 @@ async def apply_nixos_configuration( ) return False - # Save to temporary file + # SECURITY: Save to temporary file in memory-backed tmpfs + # This prevents keys from persisting on disk after crashes + shm_dir = "/dev/shm/nio-nix-keys" + os.makedirs(shm_dir, mode=0o700, exist_ok=True) + with tempfile.NamedTemporaryFile( - mode="w", prefix="ssh_key_", delete=False + mode="w", prefix="ssh_key_", delete=False, dir=shm_dir ) as tmp: tmp.write(ssh_private_key.strip() + "\n") tmp_key_path = tmp.name - os.chmod(tmp_key_path, 0o600) + # Owner read-only for additional security + os.chmod(tmp_key_path, 0o400) - # Form NIX_SSHOPTS for nixos-rebuild - nix_sshopts = f"-i {tmp_key_path} -o StrictHostKeyChecking=no" + # Form NIX_SSHOPTS for nixos-rebuild (host keys verified via known_hosts) + nix_sshopts = f"-i {tmp_key_path}" # Form argument for nixos-anywhere identity_arg_anywhere = f"-i {tmp_key_path}" @@ -295,12 +304,23 @@ async def read_stream(stream, log_func): if decoded: log_func(decoded) - # Start reading stdout and stderr in parallel - await asyncio.gather( - read_stream(process.stdout, logger.info), - read_stream(process.stderr, logger.error), - process.wait(), # wait for process completion - ) + # Start reading stdout and stderr in parallel with configurable timeout + try: + await asyncio.wait_for( + asyncio.gather( + read_stream(process.stdout, logger.info), + read_stream(process.stderr, logger.error), + process.wait(), # wait for process completion + ), + timeout=config.NIXOS_APPLY_TIMEOUT + ) + except asyncio.TimeoutError: + logger.error( + f"Command timed out after {config.NIXOS_APPLY_TIMEOUT}s: {cmd}" + ) + process.kill() + await process.wait() + return False if process.returncode != 0: logger.error(f"Command failed (code {process.returncode})") @@ -320,241 +340,107 @@ async def read_stream(stream, log_func): # ... -async def reconcile_nixos_configuration(body, spec, name, namespace, **kwargs): - """Main reconciliation point for NixosConfiguration""" +async def reconcile_nixos_configuration( + body: Dict[str, Any], spec: Dict[str, Any], name: str, namespace: str, **kwargs +) -> None: + """ + Main reconciliation point for NixosConfiguration. + + Refactored into smaller, focused functions for better maintainability. + """ logger.info(f"Reconciling NixosConfiguration: {name}") + # Import helper functions + from reconcile_helpers import ( + check_machine_availability, + prepare_git_repository, + detect_configuration_changes, + apply_and_update_status, + cleanup_repository, + ) + try: - # Check deletion timestamp deletion_timestamp = body.get("metadata", {}).get("deletionTimestamp") - # Get associated machine - machine_name = spec["machineRef"]["name"] - machine = get_machine(machine_name, namespace) - - # Check machine availability before applying configuration - is_discoverable = await check_machine_discoverable( - machine["spec"], machine_name, namespace - ) - if not is_discoverable: - logger.warning( - f"Skipping configuration application for {name}: machine {machine_name} is not discoverable due to missing credentials" - ) - # Update configuration status - await update_configuration_status( - name, - namespace, - { - "appliedCommit": None, - "lastAppliedTime": None, - "targetMachine": machine_name, - "conditions": [ - { - "type": "Applied", - "status": "False", - "lastTransitionTime": datetime.utcnow().isoformat() + "Z", - "reason": "MissingCredentials", - "message": "Configuration application skipped due to missing SSH credentials", - } - ], - }, - ) + # Step 1: Check machine availability + is_available, machine = await check_machine_availability(spec, name, namespace) + if not is_available: return - # Simple git repo handling - just use gitRepo directly - repo_url = spec["gitRepo"] - repo_name = extract_repo_name_from_url(repo_url) - - # Get git reference (branch, tag, or commit) - default to "main" - git_ref = spec.get("ref", "main") - - # Get current commit hash from the repository - new_commit_hash = await get_remote_commit_hash( - repo_url, git_ref, spec.get("credentialsRef"), namespace - ) - - # Create predictable path - workdir_path = get_workdir_path(namespace, name, repo_name, new_commit_hash) + machine_name = spec["machineRef"]["name"] - # Clone repository to predictable path - repo_path, actual_commit_hash = await clone_git_repo( - repo_url, spec.get("credentialsRef"), namespace, target_path=workdir_path + # Step 2: Prepare Git repository + repo_path, actual_commit_hash, workdir_path = await prepare_git_repository( + spec, name, namespace ) try: - # Calculate additionalFiles hash + # Step 3: Calculate hashes and detect changes additional_files_hash = get_additional_files_hash( spec, namespace, machine["spec"] ) - # Get current status for change detection - current_status = body.get("status", {}) - current_applied_commit = current_status.get("appliedCommit") - current_additional_files_hash = current_status.get( - "additionalFilesHash", "" - ) - current_has_full_install = current_status.get( - "fullDiskInstallCompleted", False - ) - - # Check if commit changed - commit_changed = current_applied_commit != actual_commit_hash - # Check if additional files changed - additional_files_changed = ( - current_additional_files_hash != additional_files_hash + should_reconcile, commit_changed, files_changed = detect_configuration_changes( + body, spec, actual_commit_hash, additional_files_hash, deletion_timestamp ) - # Check if reconciliation should be triggered - should_reconcile = False - - if deletion_timestamp: - if spec.get("onRemoveFlake"): - logger.info( - f"Deletion detected with onRemoveFlake, triggering reconcile for {name}" - ) - should_reconcile = True - else: - logger.info( - f"Deletion detected but no onRemoveFlake specified, cleaning up for {name}" - ) - # Handle deletion without onRemoveFlake - await update_machine_status( - machine_name, - namespace, - { - "hasConfiguration": False, - "appliedConfiguration": None, - "appliedCommit": None, - }, - ) - return - else: - # Check changes for normal reconciliation - if commit_changed or additional_files_changed: - logger.info( - f"Changes detected - commit: {commit_changed}, additionalFiles: {additional_files_changed}, triggering reconcile for {name}" - ) - should_reconcile = True + # Handle deletion without onRemoveFlake + if deletion_timestamp and not spec.get("onRemoveFlake"): + logger.info(f"Deletion without onRemoveFlake for {name}, cleaning up") + await update_machine_status( + machine_name, + namespace, + { + "hasConfiguration": False, + "appliedConfiguration": None, + "appliedCommit": None, + }, + ) + return if not should_reconcile: logger.info(f"No changes detected, skipping reconcile for {name}") return - # Inject additionalFiles + # Step 4: Inject additional files config_hash = await inject_additional_files( repo_path, spec, namespace, machine["spec"] ) - # Determine if fullInstall is needed (only for first time) - needs_full_install = ( - spec.get("fullInstall", False) and not current_has_full_install - ) + # Step 5: Determine if full install is needed + current_status = body.get("status", {}) + current_has_full_install = current_status.get("fullDiskInstallCompleted", False) + needs_full_install = spec.get("fullInstall", False) and not current_has_full_install - # Apply configuration - success = await apply_nixos_configuration( + # Step 6: Apply configuration and update statuses + success = await apply_and_update_status( + name, + namespace, + machine_name, machine["spec"], spec, repo_path, actual_commit_hash, - bool( - deletion_timestamp - ), # is_remove = True if deletionTimestamp exists + config_hash, + additional_files_hash, + deletion_timestamp, needs_full_install, + current_has_full_install, ) - if success: - # Update statuses - current_time = datetime.utcnow().isoformat() + "Z" - - if deletion_timestamp: - # On deletion, remove configuration from machine status - await update_machine_status( - machine_name, - namespace, - { - "hasConfiguration": False, - "appliedConfiguration": None, - "appliedCommit": None, - }, - ) - - await update_configuration_status( - name, - namespace, - { - "appliedCommit": actual_commit_hash, - "lastAppliedTime": current_time, - "targetMachine": machine_name, - "configurationHash": config_hash, - "additionalFilesHash": additional_files_hash, - "hasFullInstall": current_has_full_install, # preserve flag - "conditions": [ - { - "type": "Applied", - "status": "True", - "lastTransitionTime": current_time, - "reason": "Removed", - "message": "Configuration successfully removed", - } - ], - }, - ) - else: - # On normal reconciliation, update statuses - await update_machine_status( - machine_name, - namespace, - { - "hasConfiguration": True, - "appliedConfiguration": name, - "appliedCommit": actual_commit_hash, - "lastAppliedTime": current_time, - }, - ) - - await update_configuration_status( - name, - namespace, - { - "appliedCommit": actual_commit_hash, - "lastAppliedTime": current_time, - "targetMachine": machine_name, - "configurationHash": config_hash, - "additionalFilesHash": additional_files_hash, - "hasFullInstall": needs_full_install - or current_has_full_install, # set or preserve flag - "conditions": [ - { - "type": "Applied", - "status": "True", - "lastTransitionTime": current_time, - "reason": "Success", - "message": "Configuration successfully applied", - } - ], - }, - ) - - logger.info( - f"Successfully reconciled configuration {name} to machine {machine_name}" - ) - - # Run GC for old versions - await garbage_collect_old_versions(namespace, name, workdir_path) - - else: + if not success: raise kopf.TemporaryError("Failed to apply configuration", delay=60) finally: - # Clean up temporary directory - shutil.rmtree(repo_path, ignore_errors=True) + # Step 7: Cleanup + await cleanup_repository(repo_path, namespace, name, workdir_path) except Exception as e: - logger.error(f"Failed to reconcile NixosConfiguration {name}: {e}") + logger.error(f"Failed to reconcile NixosConfiguration {name}: {e}", exc_info=True) raise kopf.TemporaryError(f"Configuration reconciliation failed: {e}", delay=60) -async def garbage_collect_old_versions(namespace: str, name: str, current_path: str): +async def garbage_collect_old_versions(namespace: str, name: str, current_path: str) -> None: """Remove old configuration versions, keeping only current one""" base_dir = os.path.dirname(current_path) if not os.path.exists(base_dir): @@ -574,22 +460,22 @@ async def garbage_collect_old_versions(namespace: str, name: str, current_path: @kopf.on.update("nio.homystack.com", "v1alpha1", "nixosconfigurations") @kopf.on.resume("nio.homystack.com", "v1alpha1", "nixosconfigurations") @kopf.on.delete("nio.homystack.com", "v1alpha1", "nixosconfigurations") -async def unified_nixos_configuration_handler(body, spec, name, namespace, **kwargs): +async def unified_nixos_configuration_handler( + body: Dict[str, Any], spec: Dict[str, Any], name: str, namespace: str, **kwargs +) -> None: """Unified handler for all NixosConfiguration operations""" await reconcile_nixos_configuration(body, spec, name, namespace, **kwargs) -@kopf.timer("nio.homystack.com", "v1alpha1", "nixosconfigurations", interval=3600.0) -async def garbage_collect_all_old_configurations(**kwargs): +async def garbage_collect_all_old_configurations(**kwargs) -> None: """Background GC for all configurations older than 24 hours""" - base_path = "/tmp/nixos-config" - if not os.path.exists(base_path): + if not os.path.exists(config.BASE_CONFIG_PATH): return current_time = datetime.now().timestamp() - for namespace in os.listdir(base_path): - namespace_path = os.path.join(base_path, namespace) + for namespace in os.listdir(config.BASE_CONFIG_PATH): + namespace_path = os.path.join(config.BASE_CONFIG_PATH, namespace) if not os.path.isdir(namespace_path): continue diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2bf9424 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[tool.black] +line-length = 100 +target-version = ['py311'] +exclude = ''' +/( + \.git + | \.venv + | venv + | build + | dist +)/ +''' + +[tool.flake8] +max-line-length = 127 +max-complexity = 15 +exclude = [".git", "__pycache__", "build", "dist", ".venv", "venv"] +ignore = ["E203", "W503"] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..38b5561 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,41 @@ +[pytest] +# Pytest configuration for NixOS Infrastructure Operator + +# Test discovery +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Test paths +testpaths = tests + +# Async support +asyncio_mode = auto + +# Coverage options +addopts = + --verbose + --strict-markers + --cov=. + --cov-report=term-missing + --cov-report=html + --cov-report=xml + +# Markers +markers = + unit: Unit tests + integration: Integration tests + e2e: End-to-end tests + slow: Slow running tests + +# Logging +log_cli = true +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)8s] %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + +# Warnings +filterwarnings = + error + ignore::DeprecationWarning + ignore::PendingDeprecationWarning diff --git a/reconcile_helpers.py b/reconcile_helpers.py new file mode 100644 index 0000000..c95631c --- /dev/null +++ b/reconcile_helpers.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 + +""" +Helper functions for NixOS configuration reconciliation. + +This module contains smaller, focused functions that were extracted from the +original giant reconcile_nixos_configuration function for better maintainability, +testability, and code clarity. +""" + +import logging +import shutil +from datetime import datetime +from typing import Dict, Optional, Tuple, Any + +from machine_handlers import check_machine_discoverable +from clients import get_machine, update_configuration_status +from utils import ( + clone_git_repo, + get_workdir_path, + extract_repo_name_from_url, + get_remote_commit_hash, +) +from nixosconfiguration_handlers import ( + inject_additional_files, + get_additional_files_hash, + apply_nixos_configuration, + garbage_collect_old_versions, +) +from retry_utils import with_retry +import config + +logger = logging.getLogger(__name__) + + +async def check_machine_availability( + spec: Dict[str, Any], name: str, namespace: str +) -> Tuple[bool, Optional[Dict[str, Any]]]: + """ + Check if target machine is available and return machine spec. + + Args: + spec: NixosConfiguration spec + name: Configuration name + namespace: Kubernetes namespace + + Returns: + Tuple of (is_available, machine_spec) + """ + machine_name = spec["machineRef"]["name"] + machine = get_machine(machine_name, namespace) + + is_discoverable = await check_machine_discoverable( + machine["spec"], None, machine_name, namespace + ) + + if not is_discoverable: + logger.warning( + f"Machine {machine_name} is not discoverable for configuration {name}" + ) + await update_configuration_status( + name, + namespace, + { + "appliedCommit": None, + "lastAppliedTime": None, + "targetMachine": machine_name, + "conditions": [ + { + "type": "Applied", + "status": "False", + "lastTransitionTime": datetime.utcnow().isoformat() + "Z", + "reason": "MissingCredentials", + "message": "Configuration application skipped due to missing SSH credentials", + } + ], + }, + ) + return False, None + + return True, machine + + +@with_retry( + max_attempts=config.RETRY_MAX_ATTEMPTS, + initial_delay=config.RETRY_INITIAL_DELAY, + max_delay=config.RETRY_MAX_DELAY, +) +async def prepare_git_repository( + spec: Dict[str, Any], name: str, namespace: str +) -> Tuple[str, str, str]: + """ + Clone and prepare Git repository for configuration. + + Includes retry logic for transient network failures. + + Args: + spec: NixosConfiguration spec + name: Configuration name + namespace: Kubernetes namespace + + Returns: + Tuple of (repo_path, actual_commit_hash, workdir_path) + """ + repo_url = spec["gitRepo"] + repo_name = extract_repo_name_from_url(repo_url) + + # Get git reference (branch, tag, or commit) - default to "main" + git_ref = spec.get("ref", "main") + + # Get current commit hash from the repository + new_commit_hash = await get_remote_commit_hash( + repo_url, git_ref, spec.get("credentialsRef"), namespace + ) + + # Create predictable path + workdir_path = get_workdir_path(namespace, name, repo_name, new_commit_hash) + + # Clone repository to predictable path + repo_path, actual_commit_hash = await clone_git_repo( + repo_url, spec.get("credentialsRef"), namespace, target_path=workdir_path + ) + + logger.info( + f"Prepared repository {repo_name} at commit {actual_commit_hash[:8]}" + ) + return repo_path, actual_commit_hash, workdir_path + + +def detect_configuration_changes( + body: Dict[str, Any], + spec: Dict[str, Any], + actual_commit_hash: str, + additional_files_hash: str, + deletion_timestamp: Optional[str], +) -> Tuple[bool, bool, bool]: + """ + Detect if configuration has changes that require reconciliation. + + Args: + body: Full NixosConfiguration resource body + spec: NixosConfiguration spec + actual_commit_hash: Current Git commit hash + additional_files_hash: Hash of additional files + deletion_timestamp: Deletion timestamp if resource is being deleted + + Returns: + Tuple of (should_reconcile, commit_changed, additional_files_changed) + """ + current_status = body.get("status", {}) + current_applied_commit = current_status.get("appliedCommit") + current_additional_files_hash = current_status.get("additionalFilesHash", "") + + commit_changed = current_applied_commit != actual_commit_hash + additional_files_changed = current_additional_files_hash != additional_files_hash + + should_reconcile = False + + if deletion_timestamp: + if spec.get("onRemoveFlake"): + logger.info(f"Deletion with onRemoveFlake detected, will reconcile") + should_reconcile = True + else: + logger.info(f"Deletion without onRemoveFlake, will skip reconciliation") + else: + if commit_changed or additional_files_changed: + logger.info( + f"Changes detected - commit: {commit_changed}, " + f"additionalFiles: {additional_files_changed}" + ) + should_reconcile = True + + return should_reconcile, commit_changed, additional_files_changed + + +async def apply_and_update_status( + name: str, + namespace: str, + machine_name: str, + machine_spec: Dict[str, Any], + config_spec: Dict[str, Any], + repo_path: str, + actual_commit_hash: str, + config_hash: str, + additional_files_hash: str, + deletion_timestamp: Optional[str], + needs_full_install: bool, + current_has_full_install: bool, +) -> bool: + """ + Apply configuration to machine and update resource statuses. + + Args: + name: Configuration name + namespace: Kubernetes namespace + machine_name: Target machine name + machine_spec: Machine specification + config_spec: NixosConfiguration specification + repo_path: Path to cloned repository + actual_commit_hash: Git commit hash + config_hash: Configuration directory hash + additional_files_hash: Additional files hash + deletion_timestamp: If not None, resource is being deleted + needs_full_install: Whether full disk install is needed + current_has_full_install: Whether full install was done before + + Returns: + True if successful, False otherwise + """ + from clients import update_machine_status + + # Apply configuration + success = await apply_nixos_configuration( + machine_spec, + config_spec, + repo_path, + actual_commit_hash, + bool(deletion_timestamp), + needs_full_install, + ) + + if not success: + return False + + current_time = datetime.utcnow().isoformat() + "Z" + + if deletion_timestamp: + # On deletion, remove configuration from machine status + await update_machine_status( + machine_name, + namespace, + { + "hasConfiguration": False, + "appliedConfiguration": None, + "appliedCommit": None, + }, + ) + + await update_configuration_status( + name, + namespace, + { + "appliedCommit": actual_commit_hash, + "lastAppliedTime": current_time, + "targetMachine": machine_name, + "configurationHash": config_hash, + "additionalFilesHash": additional_files_hash, + "hasFullInstall": current_has_full_install, + "conditions": [ + { + "type": "Applied", + "status": "True", + "lastTransitionTime": current_time, + "reason": "Removed", + "message": "Configuration successfully removed", + } + ], + }, + ) + else: + # On normal reconciliation, update statuses + await update_machine_status( + machine_name, + namespace, + { + "hasConfiguration": True, + "appliedConfiguration": name, + "appliedCommit": actual_commit_hash, + "lastAppliedTime": current_time, + }, + ) + + await update_configuration_status( + name, + namespace, + { + "appliedCommit": actual_commit_hash, + "lastAppliedTime": current_time, + "targetMachine": machine_name, + "configurationHash": config_hash, + "additionalFilesHash": additional_files_hash, + "hasFullInstall": needs_full_install or current_has_full_install, + "conditions": [ + { + "type": "Applied", + "status": "True", + "lastTransitionTime": current_time, + "reason": "Success", + "message": "Configuration successfully applied", + } + ], + }, + ) + + logger.info(f"Successfully reconciled configuration {name} to machine {machine_name}") + return True + + +async def cleanup_repository(repo_path: str, namespace: str, name: str, workdir_path: str) -> None: + """ + Clean up repository and run garbage collection. + + Args: + repo_path: Path to repository to clean up + namespace: Kubernetes namespace + name: Configuration name + workdir_path: Current working directory path + """ + try: + shutil.rmtree(repo_path, ignore_errors=True) + await garbage_collect_old_versions(namespace, name, workdir_path) + except Exception as e: + logger.warning(f"Cleanup failed: {e}") diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..d6326cb --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,13 @@ +# Development dependencies +-r requirements.txt + +# Testing +pytest>=7.4.0 +pytest-asyncio>=0.21.0 +pytest-cov>=4.1.0 +pytest-mock>=3.11.0 + +# Code quality +black>=23.7.0 +flake8>=6.1.0 +mypy>=1.5.0 diff --git a/requirements.txt b/requirements.txt index 4f435b4..57cd0a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ kubernetes>=26.1.0 gitpython>=3.1.0 asyncssh>=2.14.0 pyyaml>=6.0 -asyncio +prometheus-client>=0.19.0 +aiohttp>=3.9.0 fastapi uvicorn diff --git a/retry_utils.py b/retry_utils.py new file mode 100644 index 0000000..0ab7a32 --- /dev/null +++ b/retry_utils.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 + +""" +Retry utilities with exponential backoff for transient failures. + +This module provides decorators and functions for implementing retry logic +with exponential backoff and jitter to handle temporary network issues, +API rate limits, and other transient failures. +""" + +import asyncio +import logging +import random +from typing import TypeVar, Callable, Any, Optional +from functools import wraps + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + + +class RetryExhaustedError(Exception): + """Raised when all retry attempts have been exhausted""" + + pass + + +async def retry_with_backoff( + func: Callable[..., T], + *args, + max_attempts: int = 5, + initial_delay: float = 1.0, + max_delay: float = 60.0, + exponential_base: float = 2.0, + jitter: bool = True, + exceptions: tuple = (Exception,), + **kwargs, +) -> T: + """ + Retry a function with exponential backoff. + + Args: + func: Function to retry (can be sync or async) + *args: Positional arguments to pass to func + max_attempts: Maximum number of retry attempts + initial_delay: Initial delay between retries in seconds + max_delay: Maximum delay between retries in seconds + exponential_base: Base for exponential backoff calculation + jitter: Whether to add random jitter to delay + exceptions: Tuple of exceptions to catch and retry on + **kwargs: Keyword arguments to pass to func + + Returns: + Result of successful function call + + Raises: + RetryExhaustedError: If all retry attempts are exhausted + Exception: The last exception if retries are exhausted + """ + last_exception = None + + for attempt in range(1, max_attempts + 1): + try: + # Call function (handle both sync and async) + if asyncio.iscoroutinefunction(func): + result = await func(*args, **kwargs) + else: + result = func(*args, **kwargs) + + if attempt > 1: + logger.info(f"Function {func.__name__} succeeded on attempt {attempt}") + + return result + + except exceptions as e: + last_exception = e + + if attempt == max_attempts: + logger.error( + f"Function {func.__name__} failed after {max_attempts} attempts: {e}" + ) + break + + # Calculate delay with exponential backoff + delay = min(initial_delay * (exponential_base ** (attempt - 1)), max_delay) + + # Add jitter to prevent thundering herd + if jitter: + delay = delay * (0.5 + random.random()) + + logger.warning( + f"Function {func.__name__} failed on attempt {attempt}/{max_attempts}: {e}. " + f"Retrying in {delay:.2f}s..." + ) + + await asyncio.sleep(delay) + + # All retries exhausted + raise RetryExhaustedError( + f"Function {func.__name__} failed after {max_attempts} attempts" + ) from last_exception + + +def with_retry( + max_attempts: int = 5, + initial_delay: float = 1.0, + max_delay: float = 60.0, + exponential_base: float = 2.0, + jitter: bool = True, + exceptions: tuple = (Exception,), +): + """ + Decorator to add retry logic with exponential backoff to async functions. + + Example: + @with_retry(max_attempts=3, initial_delay=2.0) + async def fetch_data(): + # ... network call that might fail + pass + + Args: + max_attempts: Maximum number of retry attempts + initial_delay: Initial delay between retries in seconds + max_delay: Maximum delay between retries in seconds + exponential_base: Base for exponential backoff calculation + jitter: Whether to add random jitter to delay + exceptions: Tuple of exceptions to catch and retry on + + Returns: + Decorated function with retry logic + """ + + def decorator(func: Callable) -> Callable: + @wraps(func) + async def wrapper(*args, **kwargs): + return await retry_with_backoff( + func, + *args, + max_attempts=max_attempts, + initial_delay=initial_delay, + max_delay=max_delay, + exponential_base=exponential_base, + jitter=jitter, + exceptions=exceptions, + **kwargs, + ) + + return wrapper + + return decorator + + +class RetryableOperation: + """ + Context manager for retryable operations with statistics tracking. + + Example: + async with RetryableOperation("fetch_config", max_attempts=3) as op: + data = await fetch_from_api() + op.success() + """ + + def __init__( + self, + operation_name: str, + max_attempts: int = 5, + initial_delay: float = 1.0, + max_delay: float = 60.0, + jitter: bool = True, + ): + self.operation_name = operation_name + self.max_attempts = max_attempts + self.initial_delay = initial_delay + self.max_delay = max_delay + self.jitter = jitter + self.attempt = 0 + self.succeeded = False + + async def __aenter__(self): + self.attempt += 1 + logger.debug(f"Starting {self.operation_name} (attempt {self.attempt}/{self.max_attempts})") + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + if exc_type is None: + self.succeeded = True + if self.attempt > 1: + logger.info( + f"{self.operation_name} succeeded on attempt {self.attempt}" + ) + return True + + if self.attempt < self.max_attempts: + delay = min( + self.initial_delay * (2 ** (self.attempt - 1)), self.max_delay + ) + if self.jitter: + delay = delay * (0.5 + random.random()) + + logger.warning( + f"{self.operation_name} failed (attempt {self.attempt}/{self.max_attempts}): {exc_val}. " + f"Retrying in {delay:.2f}s..." + ) + await asyncio.sleep(delay) + return False # Suppress exception to allow retry + + logger.error( + f"{self.operation_name} failed after {self.max_attempts} attempts: {exc_val}" + ) + return False # Re-raise the exception + + def success(self): + """Mark operation as successful""" + self.succeeded = True + + def retry(self, exception: Exception): + """Explicitly trigger a retry by raising an exception. + + This method is useful when you want to manually control retry logic + from within the context manager block. + + Args: + exception: The exception that triggered the retry + + Raises: + The provided exception, which will be handled by __aexit__ + """ + raise exception diff --git a/ssh_utils.py b/ssh_utils.py new file mode 100644 index 0000000..658e15a --- /dev/null +++ b/ssh_utils.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 + +import logging +import asyncssh +import tempfile +import os +from typing import Dict, Optional, Tuple, Any +from clients import get_secret_data +from events import emit_missing_credentials_event +from known_hosts_manager import get_known_hosts_manager +from input_validation import validate_hostname, validate_ssh_username, ValidationError + +logger = logging.getLogger(__name__) + + +async def establish_ssh_connection( + machine_spec: Dict[str, Any], + body: Optional[Dict[str, Any]] = None, + machine_name: Optional[str] = None, + namespace: Optional[str] = None, +) -> Tuple[Optional[asyncssh.SSHClientConnection], Optional[str]]: + """ + Establish SSH connection to a machine using key, password, or no authentication. + + Uses Trust On First Use (TOFU) policy for host key verification. + + Returns: + Tuple of (connection, temp_key_path) where connection is the SSH connection + and temp_key_path is the path to temporary SSH key file (if created, None otherwise). + Returns (None, None) if connection fails. + """ + # SECURITY: Validate inputs to prevent command injection + try: + hostname = validate_hostname(machine_spec["hostname"]) + username = validate_ssh_username(machine_spec.get("sshUser", "root")) + except ValidationError as e: + logger.error(f"Input validation failed: {e}") + return None, None + + # Get known_hosts manager for host verification + known_hosts_mgr = get_known_hosts_manager() + + ssh_config = { + "host": hostname, + "username": username, + "known_hosts": known_hosts_mgr.get_known_hosts_path(), # Enable host verification + } + + has_credentials = False + ssh_key_temp_file = None + + # Attempt SSH key connection + if "sshKeySecretRef" in machine_spec: + try: + secret_data = await get_secret_data( + machine_spec["sshKeySecretRef"]["name"], + machine_spec["sshKeySecretRef"].get("namespace", "default"), + ) + if "ssh-privatekey" in secret_data and secret_data["ssh-privatekey"]: + # SECURITY: Create temporary file in memory-backed tmpfs (/dev/shm) + # This prevents keys from being written to disk and persisting after crashes + shm_dir = "/dev/shm/nio-ssh-keys" + os.makedirs(shm_dir, mode=0o700, exist_ok=True) + + with tempfile.NamedTemporaryFile( + mode="w", + delete=False, + suffix="_ssh_key", + dir=shm_dir, # Use memory-backed tmpfs + ) as temp_file: + temp_file.write(secret_data["ssh-privatekey"]) + ssh_key_temp_file = temp_file.name + + # Set correct permissions for SSH key (owner read-only) + os.chmod(ssh_key_temp_file, 0o400) + + ssh_config["client_keys"] = [ssh_key_temp_file] + has_credentials = True + logger.info("Using SSH key for authentication") + else: + # Secret exists but doesn't contain SSH key + if body: + emit_missing_credentials_event( + body, + "MissingSSHKey", + f"Secret {machine_spec['sshKeySecretRef']['name']} exists but doesn't contain 'ssh-privatekey'", + ) + logger.warning( + f"Secret {machine_spec['sshKeySecretRef']['name']} exists but doesn't contain 'ssh-privatekey'" + ) + except Exception as e: + # Secret not found or unavailable + if body: + emit_missing_credentials_event( + body, + "SecretNotFound", + f"Failed to get SSH key from secret {machine_spec['sshKeySecretRef']['name']}", + ) + logger.warning( + f"Failed to get SSH key from secret {machine_spec['sshKeySecretRef']['name']}: {e}" + ) + + # Attempt password connection (if key didn't work or not specified) + if not has_credentials and "sshPasswordSecretRef" in machine_spec: + try: + secret_data = await get_secret_data( + machine_spec["sshPasswordSecretRef"]["name"], + machine_spec["sshPasswordSecretRef"].get("namespace", "default"), + ) + + # Determine password key (default 'password') + password_key = machine_spec["sshPasswordSecretRef"].get( + "key", "password" + ) + + if password_key in secret_data and secret_data[password_key]: + ssh_config["password"] = secret_data[password_key] + has_credentials = True + logger.info("Using password for authentication") + else: + # Secret exists but doesn't contain password + if body: + emit_missing_credentials_event( + body, + "MissingPassword", + f"Secret {machine_spec['sshPasswordSecretRef']['name']} exists but doesn't contain '{password_key}'", + ) + logger.warning( + f"Secret {machine_spec['sshPasswordSecretRef']['name']} exists but doesn't contain '{password_key}'" + ) + except Exception as e: + # Secret not found or unavailable + if body: + emit_missing_credentials_event( + body, + "SecretNotFound", + f"Failed to get password from secret {machine_spec['sshPasswordSecretRef']['name']}", + ) + logger.warning( + f"Failed to get password from secret {machine_spec['sshPasswordSecretRef']['name']}: {e}" + ) + + # If no credentials provided, try connection without authentication + if not has_credentials: + logger.info( + "No SSH key or password provided, attempting connection without authentication" + ) + + # Attempt connection + try: + conn = await asyncssh.connect(**ssh_config) + return conn, ssh_key_temp_file + except Exception as e: + logger.warning( + f"Machine {machine_spec.get('hostname')} connection failed: {e}" + ) + # Clean up temp file if connection failed + if ssh_key_temp_file and os.path.exists(ssh_key_temp_file): + try: + os.unlink(ssh_key_temp_file) + except Exception as cleanup_error: + logger.warning( + f"Failed to delete temporary SSH key file {ssh_key_temp_file}: {cleanup_error}" + ) + return None, None + + +def cleanup_ssh_key(ssh_key_temp_file: Optional[str]) -> None: + """Clean up temporary SSH key file""" + if ssh_key_temp_file and os.path.exists(ssh_key_temp_file): + try: + os.unlink(ssh_key_temp_file) + logger.debug(f"Deleted temporary SSH key file: {ssh_key_temp_file}") + except Exception as e: + logger.warning( + f"Failed to delete temporary SSH key file {ssh_key_temp_file}: {e}" + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e86665a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Unit tests for NixOS Infrastructure Operator.""" diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..231f13a --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +"""Unit tests for configuration module.""" + +import os +import pytest + + +class TestConfigurationDefaults: + """Tests for configuration default values.""" + + def test_default_values_exist(self): + """All configuration values should have defaults.""" + # Import fresh to get defaults + import importlib + import config as config_module + + importlib.reload(config_module) + + # Check that all expected config values exist + assert hasattr(config_module, "BASE_CONFIG_PATH") + assert hasattr(config_module, "KNOWN_HOSTS_PATH") + assert hasattr(config_module, "REMOTE_HARDWARE_SCRIPT_PATH") + assert hasattr(config_module, "MACHINE_DISCOVERY_INTERVAL") + assert hasattr(config_module, "HARDWARE_SCAN_INTERVAL") + assert hasattr(config_module, "CONFIG_RECONCILE_INTERVAL") + assert hasattr(config_module, "NIXOS_APPLY_TIMEOUT") + assert hasattr(config_module, "RETRY_MAX_ATTEMPTS") + assert hasattr(config_module, "RETRY_INITIAL_DELAY") + assert hasattr(config_module, "RETRY_MAX_DELAY") + assert hasattr(config_module, "RETRY_EXPONENTIAL_BASE") + assert hasattr(config_module, "METRICS_PORT") + + def test_intervals_are_positive(self): + """Interval values should be positive.""" + import config + + assert config.MACHINE_DISCOVERY_INTERVAL > 0 + assert config.HARDWARE_SCAN_INTERVAL > 0 + assert config.CONFIG_RECONCILE_INTERVAL > 0 + + def test_timeout_is_positive(self): + """Timeout value should be positive.""" + import config + + assert config.NIXOS_APPLY_TIMEOUT > 0 + + def test_retry_config_is_valid(self): + """Retry configuration should be valid.""" + import config + + assert config.RETRY_MAX_ATTEMPTS >= 1 + assert config.RETRY_INITIAL_DELAY > 0 + assert config.RETRY_MAX_DELAY >= config.RETRY_INITIAL_DELAY + assert config.RETRY_EXPONENTIAL_BASE > 1.0 + + def test_metrics_port_is_valid(self): + """Metrics port should be valid.""" + import config + + assert 1 <= config.METRICS_PORT <= 65535 + + +class TestConfigurationEnvironmentOverride: + """Tests for environment variable overrides.""" + + def test_env_override_string(self): + """String config can be overridden by environment.""" + os.environ["NIO_BASE_CONFIG_PATH"] = "/custom/path" + + import importlib + import config as config_module + + importlib.reload(config_module) + + assert config_module.BASE_CONFIG_PATH == "/custom/path" + + # Cleanup + del os.environ["NIO_BASE_CONFIG_PATH"] + + def test_env_override_int(self): + """Integer config can be overridden by environment.""" + os.environ["NIO_RETRY_MAX_ATTEMPTS"] = "10" + + import importlib + import config as config_module + + importlib.reload(config_module) + + assert config_module.RETRY_MAX_ATTEMPTS == 10 + + # Cleanup + del os.environ["NIO_RETRY_MAX_ATTEMPTS"] + + def test_env_override_float(self): + """Float config can be overridden by environment.""" + os.environ["NIO_RETRY_INITIAL_DELAY"] = "5.5" + + import importlib + import config as config_module + + importlib.reload(config_module) + + assert config_module.RETRY_INITIAL_DELAY == 5.5 + + # Cleanup + del os.environ["NIO_RETRY_INITIAL_DELAY"] + + +class TestConfigSummary: + """Tests for configuration summary.""" + + def test_config_summary_format(self): + """Config summary should be properly formatted.""" + import config + + summary = config.get_config_summary() + + # Should be a string + assert isinstance(summary, str) + + # Should contain key configuration items + assert "Base config path" in summary + assert "Machine discovery" in summary + assert "Retry" in summary + assert "Metrics" in summary diff --git a/tests/test_e2e_operator.py b/tests/test_e2e_operator.py new file mode 100644 index 0000000..4eee3f7 --- /dev/null +++ b/tests/test_e2e_operator.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 + +"""End-to-end tests for NixOS Infrastructure Operator. + +These tests run the full operator workflow: +1. Create kind cluster +2. Deploy operator +3. Create Machine and NixOSConfiguration resources +4. Verify reconciliation with mock SSH server +5. Cleanup +""" + +import pytest +import asyncio +import asyncssh +import tempfile +import os + + +pytestmark = pytest.mark.e2e + + +class MockSSHServerAuth(asyncssh.SSHServer): + """SSH Server auth handler.""" + + def __init__(self, test_password="test123"): + self.test_password = test_password + + def begin_auth(self, username): + """Begin authentication for a user.""" + return True + + def password_auth_supported(self): + """Enable password authentication.""" + return True + + def validate_password(self, username, password): + """Validate password.""" + return password == self.test_password + + +class MockSSHServer: + """Mock SSH server for E2E testing.""" + + def __init__(self, port=2222): + """Initialize mock SSH server.""" + self.port = port + self.server = None + self.host_key = None + self.commands_executed = [] + self.test_password = "test123" + + async def start(self): + """Start the mock SSH server.""" + # Generate host key + self.host_key = asyncssh.generate_private_key("ssh-rsa") + + # Start server with password auth + self.server = await asyncssh.listen( + "localhost", + self.port, + server_host_keys=[self.host_key], + server_factory=lambda: MockSSHServerAuth(self.test_password), + process_factory=self.handle_client, + reuse_address=True, # Allow reuse of address to avoid port conflicts + ) + + def get_password(self): + """Get the test password for connections.""" + return self.test_password + + async def handle_client(self, process): + """Handle client commands.""" + command = process.command + self.commands_executed.append(command) + + # Mock responses for common commands + if command == "uname -a": + process.stdout.write("Linux test 6.1.0 NixOS\n") + elif command == "nixos-version": + process.stdout.write("23.11\n") + elif "nixos-rebuild" in command: + process.stdout.write("building system configuration...\n") + await asyncio.sleep(0.1) + process.stdout.write("activation finished successfully\n") + elif command.startswith("cat /etc/machine-id"): + process.stdout.write("test-machine-id-12345\n") + else: + process.stdout.write(f"Mock command: {command}\n") + + process.exit(0) + + async def stop(self): + """Stop the mock SSH server.""" + if self.server: + self.server.close() + await self.server.wait_closed() + + def get_executed_commands(self): + """Get list of executed commands.""" + return self.commands_executed + + +class TestE2EBasicWorkflow: + """E2E tests for basic operator workflow.""" + + @pytest.fixture + async def mock_ssh_server(self): + """Start mock SSH server for tests.""" + server = MockSSHServer(port=2222) + await server.start() + yield server + await server.stop() + # Give asyncssh time to fully clean up the server + await asyncio.sleep(0.1) + + @pytest.mark.asyncio + @pytest.mark.skipif(True, reason="Requires kind cluster setup") + async def test_full_operator_workflow(self, mock_ssh_server): + """Test complete operator workflow with mock SSH server.""" + # This test would require: + # 1. kind cluster creation + # 2. CRD installation + # 3. Operator deployment + # 4. Resource creation + # 5. Reconciliation verification + # 6. Cleanup + + # For now, this is a placeholder + # Real implementation would use kubernetes client to interact with cluster + pass + + @pytest.mark.asyncio + async def test_ssh_connection_mock(self, mock_ssh_server): + """Test SSH connection to mock server.""" + # Connect to mock server with password + async with asyncssh.connect( + "localhost", + port=2222, + username="test", + password=mock_ssh_server.get_password(), + known_hosts=None, # Accept any host key for testing + ) as conn: + # Execute test command + result = await conn.run("uname -a") + assert result.exit_status == 0 + assert "NixOS" in result.stdout + + @pytest.mark.asyncio + async def test_nixos_rebuild_mock(self, mock_ssh_server): + """Test NixOS rebuild command execution.""" + async with asyncssh.connect( + "localhost", + port=2222, + username="test", + password=mock_ssh_server.get_password(), + known_hosts=None, + ) as conn: + # Execute nixos-rebuild command + result = await conn.run("nixos-rebuild switch") + assert result.exit_status == 0 + assert "activation finished" in result.stdout + + # Verify command was executed + commands = mock_ssh_server.get_executed_commands() + assert any("nixos-rebuild" in cmd for cmd in commands) + + +class TestE2EMachineDiscovery: + """E2E tests for machine discovery workflow.""" + + @pytest.fixture + async def mock_ssh_server(self): + """Start mock SSH server.""" + server = MockSSHServer(port=2223) + await server.start() + yield server + await server.stop() + # Give asyncssh time to fully clean up the server + await asyncio.sleep(0.1) + + @pytest.mark.asyncio + async def test_machine_discoverable_check(self, mock_ssh_server): + """Test machine discoverability check via SSH.""" + # Note: Full implementation would import and use check_machine_discoverable + # from machine_handlers, but for now we just verify SSH connectivity + + # For now, just verify mock server is running + async with asyncssh.connect( + "localhost", + port=2223, + username="test", + password=mock_ssh_server.get_password(), + known_hosts=None, + ) as conn: + result = await conn.run("echo test") + assert result.exit_status == 0 + + +class TestE2EHardwareScanning: + """E2E tests for hardware scanning workflow.""" + + @pytest.fixture + async def mock_ssh_server_with_hardware(self): + """Mock SSH server that returns hardware info.""" + + class HardwareSSHServer(MockSSHServer): + async def handle_client(self, process): + """Handle hardware scanning commands.""" + command = process.command + self.commands_executed.append(command) + + if "hardware_scanner.sh" in command or "lscpu" in command: + # Mock hardware output + hardware_json = '{"cpu": "8", "memory": "16GB", "disk": "500GB"}' + process.stdout.write(hardware_json) + process.exit(0) + else: + await super().handle_client(process) + + server = HardwareSSHServer(port=2225) + await server.start() + yield server + await server.stop() + # Give asyncssh time to fully clean up the server + await asyncio.sleep(0.1) + + @pytest.mark.asyncio + async def test_hardware_scan_execution(self, mock_ssh_server_with_hardware): + """Test hardware scanning returns data.""" + async with asyncssh.connect( + "localhost", + port=2225, + username="test", + password=mock_ssh_server_with_hardware.get_password(), + known_hosts=None, + ) as conn: + # Execute hardware scan + result = await conn.run("lscpu") + assert result.exit_status == 0 + assert "cpu" in result.stdout.lower() + + +class TestE2EGitOperations: + """E2E tests for Git repository operations.""" + + @pytest.mark.asyncio + async def test_git_clone_real_repo(self): + """Test cloning a real public repository.""" + from utils import clone_git_repo + + with tempfile.TemporaryDirectory() as tmpdir: + # Clone a small public repo for testing + repo_url = "https://github.com/NixOS/templates.git" + + try: + repo_path, commit_hash = await clone_git_repo( + repo_url, None, "default", target_path=tmpdir + "/repo" + ) + + # Verify clone succeeded + assert os.path.exists(repo_path) + assert len(commit_hash) == 40 # Git SHA-1 hash length + assert os.path.exists(os.path.join(repo_path, ".git")) + except Exception as e: + # Network issues might cause this to fail in CI + pytest.skip(f"Git clone failed (network issue?): {e}") + + @pytest.mark.asyncio + async def test_git_commit_hash_retrieval(self): + """Test retrieving commit hash from repository.""" + from utils import get_remote_commit_hash + + try: + # Get commit hash for main branch + commit_hash = await get_remote_commit_hash( + "https://github.com/NixOS/templates.git", "main", None, "default" + ) + + # Verify we got a valid commit hash + assert len(commit_hash) == 40 + assert all(c in "0123456789abcdef" for c in commit_hash.lower()) + except Exception as e: + pytest.skip(f"Failed to get commit hash (network issue?): {e}") diff --git a/tests/test_events.py b/tests/test_events.py new file mode 100644 index 0000000..48ecfc3 --- /dev/null +++ b/tests/test_events.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +"""Unit tests for Kubernetes event emission module.""" + +import pytest +from unittest.mock import patch, MagicMock +from events import ( + emit_missing_credentials_event, + emit_configuration_applied_event, + emit_error_event, +) + + +class TestEventEmission: + """Tests for event emission functions.""" + + def test_emit_missing_credentials_event_success(self): + """Should emit warning event for missing credentials.""" + body = {"metadata": {"name": "test-machine"}} + reason = "MissingCredentials" + message = "SSH credentials not found" + + with patch("events.kopf.warn") as mock_warn, patch( + "events.logger" + ) as mock_logger: + emit_missing_credentials_event(body, reason, message) + + mock_warn.assert_called_once_with( + body, reason=reason, message=message + ) + mock_logger.warning.assert_called_once() + + def test_emit_missing_credentials_event_failure(self): + """Should handle kopf.warn failure gracefully.""" + body = {"metadata": {"name": "test-machine"}} + reason = "MissingCredentials" + message = "SSH credentials not found" + + with patch("events.kopf.warn", side_effect=Exception("Kopf error")), patch( + "events.logger" + ) as mock_logger: + emit_missing_credentials_event(body, reason, message) + + mock_logger.error.assert_called_once() + assert "Failed to emit missing credentials event" in str( + mock_logger.error.call_args + ) + + def test_emit_configuration_applied_event_success(self): + """Should emit info event for configuration applied.""" + body = {"metadata": {"name": "test-config"}} + reason = "ConfigurationApplied" + message = "NixOS configuration successfully applied" + + with patch("events.kopf.info") as mock_info, patch( + "events.logger" + ) as mock_logger: + emit_configuration_applied_event(body, reason, message) + + mock_info.assert_called_once_with( + body, reason=reason, message=message + ) + mock_logger.info.assert_called_once() + + def test_emit_configuration_applied_event_failure(self): + """Should handle kopf.info failure gracefully.""" + body = {"metadata": {"name": "test-config"}} + reason = "ConfigurationApplied" + message = "NixOS configuration successfully applied" + + with patch("events.kopf.info", side_effect=Exception("Kopf error")), patch( + "events.logger" + ) as mock_logger: + emit_configuration_applied_event(body, reason, message) + + mock_logger.error.assert_called_once() + assert "Failed to emit configuration applied event" in str( + mock_logger.error.call_args + ) + + def test_emit_error_event_success(self): + """Should emit exception event for errors.""" + body = {"metadata": {"name": "test-machine"}} + reason = "BuildFailed" + message = "NixOS build failed: syntax error" + + with patch("events.kopf.exception") as mock_exception, patch( + "events.logger" + ) as mock_logger: + emit_error_event(body, reason, message) + + mock_exception.assert_called_once_with( + body, reason=reason, message=message + ) + mock_logger.error.assert_called_once() + + def test_emit_error_event_failure(self): + """Should handle kopf.exception failure gracefully.""" + body = {"metadata": {"name": "test-machine"}} + reason = "BuildFailed" + message = "NixOS build failed: syntax error" + + with patch("events.kopf.exception", side_effect=Exception("Kopf error")), patch( + "events.logger" + ) as mock_logger: + emit_error_event(body, reason, message) + + # Should log the failed emission + mock_logger.error.assert_called_once() + assert "Failed to emit error event" in str(mock_logger.error.call_args) + + +class TestEventMessageFormats: + """Tests for event message formatting and logging.""" + + def test_missing_credentials_message_logged(self): + """Missing credentials event message should be logged.""" + body = {"metadata": {"name": "machine1"}} + message = "Custom credentials message" + + with patch("events.kopf.warn"), patch("events.logger") as mock_logger: + emit_missing_credentials_event(body, "Reason", message) + + call_args = str(mock_logger.warning.call_args) + assert "Emitted missing credentials event" in call_args + assert message in call_args + + def test_configuration_applied_message_logged(self): + """Configuration applied event message should be logged.""" + body = {"metadata": {"name": "config1"}} + message = "Custom config message" + + with patch("events.kopf.info"), patch("events.logger") as mock_logger: + emit_configuration_applied_event(body, "Reason", message) + + call_args = str(mock_logger.info.call_args) + assert "Emitted configuration applied event" in call_args + assert message in call_args + + def test_error_event_message_logged(self): + """Error event message should be logged.""" + body = {"metadata": {"name": "machine1"}} + message = "Custom error message" + + with patch("events.kopf.exception"), patch("events.logger") as mock_logger: + emit_error_event(body, "Reason", message) + + call_args = str(mock_logger.error.call_args) + assert "Emitted error event" in call_args + assert message in call_args diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..34e708b --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +"""Unit tests for health check server module.""" + +import pytest +from unittest.mock import Mock +from aiohttp import web +from health import HealthCheckServer, run_health_server + + +@pytest.mark.asyncio +class TestHealthCheckServer: + """Tests for HealthCheckServer class.""" + + async def test_initialization(self): + """Server should initialize with default host and port.""" + server = HealthCheckServer() + assert server.host == "0.0.0.0" + assert server.port == 8080 + assert server._is_ready is False + assert server.runner is None + + async def test_initialization_custom_port(self): + """Server should accept custom host and port.""" + server = HealthCheckServer(host="127.0.0.1", port=9090) + assert server.host == "127.0.0.1" + assert server.port == 9090 + + async def test_health_handler(self): + """Health handler should return 200 with healthy status.""" + server = HealthCheckServer() + request = Mock(spec=web.Request) + response = await server.health_handler(request) + assert response.status == 200 + assert response.content_type == "application/json" + + async def test_readiness_handler_not_ready(self): + """Readiness handler should return 503 when not ready.""" + server = HealthCheckServer() + request = Mock(spec=web.Request) + response = await server.readiness_handler(request) + assert response.status == 503 + + async def test_readiness_handler_ready(self): + """Readiness handler should return 200 when ready.""" + server = HealthCheckServer() + server.mark_ready() + request = Mock(spec=web.Request) + response = await server.readiness_handler(request) + assert response.status == 200 + + async def test_liveness_handler(self): + """Liveness handler should always return 200.""" + server = HealthCheckServer() + request = Mock(spec=web.Request) + response = await server.liveness_handler(request) + assert response.status == 200 + + async def test_mark_ready(self): + """mark_ready should set _is_ready to True.""" + server = HealthCheckServer() + assert server._is_ready is False + server.mark_ready() + assert server._is_ready is True + + async def test_mark_not_ready(self): + """mark_not_ready should set _is_ready to False.""" + server = HealthCheckServer() + server.mark_ready() + assert server._is_ready is True + server.mark_not_ready() + assert server._is_ready is False + + async def test_start_and_stop(self): + """Server should start and stop gracefully.""" + server = HealthCheckServer(host="127.0.0.1", port=18080) + try: + # Start server + await server.start() + assert server.runner is not None + + # Stop server + await server.stop() + except OSError as e: + # Port might be in use, that's ok for this test + pytest.skip(f"Port binding failed: {e}") + + async def test_run_health_server(self): + """run_health_server should create and start server.""" + try: + server = await run_health_server(host="127.0.0.1", port=18081) + assert isinstance(server, HealthCheckServer) + assert server.runner is not None + await server.stop() + except OSError as e: + pytest.skip(f"Port binding failed: {e}") + + +@pytest.mark.asyncio +class TestHealthCheckEndpoints: + """Integration tests for health check endpoints.""" + + async def test_routes_configured(self): + """Server should have all routes configured.""" + server = HealthCheckServer() + # Get all resources and extract paths + paths = [] + for resource in server.app.router.resources(): + paths.append(resource.canonical) + assert "/health" in paths + assert "/ready" in paths + assert "/live" in paths + + async def test_ready_state_transitions(self): + """Server should handle ready state transitions correctly.""" + server = HealthCheckServer() + + # Initially not ready + assert server._is_ready is False + + # Mark ready + server.mark_ready() + assert server._is_ready is True + + # Mark not ready + server.mark_not_ready() + assert server._is_ready is False + + # Mark ready again + server.mark_ready() + assert server._is_ready is True diff --git a/tests/test_input_validation.py b/tests/test_input_validation.py new file mode 100644 index 0000000..051c96a --- /dev/null +++ b/tests/test_input_validation.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 + +"""Unit tests for input validation module.""" + +import pytest +from input_validation import ( + validate_hostname, + validate_git_url, + validate_ssh_username, + validate_path, + ValidationError, +) + + +class TestValidateHostname: + """Tests for hostname validation.""" + + def test_valid_hostnames(self): + """Valid hostnames should pass validation.""" + valid = [ + "example.com", + "subdomain.example.com", + "192.168.1.1", + "localhost", + "host-name", + "host.name.with.dots", + "[::1]", # IPv6 + "[2001:db8::1]", # IPv6 + ] + for hostname in valid: + result = validate_hostname(hostname) + assert result == hostname + + def test_invalid_hostnames(self): + """Invalid hostnames should raise ValidationError.""" + invalid = [ + "host name", # space + "host;name", # semicolon + "host&name", # ampersand + "host|name", # pipe + "host`name", # backtick + "host$name", # dollar + "host(name)", # parentheses + "", # empty + ] + for hostname in invalid: + with pytest.raises(ValidationError): + validate_hostname(hostname) + + +class TestValidateGitUrl: + """Tests for Git URL validation.""" + + def test_valid_git_urls(self): + """Valid Git URLs should pass validation.""" + valid = [ + "https://github.com/owner/repo.git", + "https://gitlab.com/owner/repo.git", + "git@github.com:owner/repo.git", + "ssh://git@github.com/owner/repo.git", + ] + for url in valid: + result = validate_git_url(url) + assert isinstance(result, str) + + def test_invalid_git_urls(self): + """Invalid Git URLs should raise ValidationError.""" + invalid = [ + "ftp://malicious.com/repo.git", # wrong protocol + "https://github.com/owner/repo.git; rm -rf /", # injection + "file:///etc/passwd", # file protocol + "", # empty + ] + for url in invalid: + with pytest.raises(ValidationError): + validate_git_url(url) + + +class TestValidateUsername: + """Tests for username validation.""" + + def test_valid_usernames(self): + """Valid usernames should pass validation.""" + valid = [ + "root", + "user123", + "deploy-user", + "user_name", + "a", # single char + ] + for username in valid: + result = validate_ssh_username(username) + assert result == username + + def test_invalid_usernames(self): + """Invalid usernames should raise ValidationError.""" + invalid = [ + "user name", # space + "user;name", # semicolon + "user&name", # ampersand + "user|name", # pipe + "user`name", # backtick + "user$name", # dollar + "", # empty + ] + for username in invalid: + with pytest.raises(ValidationError): + validate_ssh_username(username) + + +class TestValidatePath: + """Tests for path validation.""" + + def test_valid_paths(self): + """Valid paths should pass validation.""" + valid = [ + "/etc/nixos/configuration.nix", + "/home/user/file.txt", + "/tmp/test", + "relative/path/file.txt", + "./file.txt", + ] + for path in valid: + result = validate_path(path) + assert result == path + + def test_invalid_paths(self): + """Invalid paths should raise ValidationError.""" + invalid = [ + "/etc/nixos/config; rm -rf /", # injection + "/etc/passwd && cat /etc/shadow", # injection + "/tmp/`whoami`", # command substitution + "/tmp/$(whoami)", # command substitution + "", # empty + ] + for path in invalid: + with pytest.raises(ValidationError): + validate_path(path) diff --git a/tests/test_integration_k8s.py b/tests/test_integration_k8s.py new file mode 100644 index 0000000..83e576e --- /dev/null +++ b/tests/test_integration_k8s.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 + +"""Integration tests for Kubernetes API interactions. + +These tests require a Kubernetes cluster (kind, minikube, or real cluster). +They test the actual API interactions, not mocked responses. +""" + +import pytest +from kubernetes import client, config as k8s_config +from kubernetes.client.rest import ApiException + + +pytestmark = pytest.mark.integration + + +class TestKubernetesConnectivity: + """Tests for basic Kubernetes connectivity.""" + + @pytest.fixture(scope="class", autouse=True) + def setup_k8s(self): + """Setup Kubernetes client for tests.""" + try: + k8s_config.load_kube_config() + except Exception: + pytest.skip("No Kubernetes cluster available") + + def test_api_server_reachable(self): + """Kubernetes API server should be reachable.""" + v1 = client.CoreV1Api() + # Simple API call to verify connectivity + namespaces = v1.list_namespace() + assert namespaces is not None + assert len(namespaces.items) > 0 + + def test_custom_resources_api_available(self): + """Custom resources API should be available.""" + api = client.CustomObjectsApi() + assert api is not None + + +class TestMachineResourceOperations: + """Integration tests for Machine custom resource operations.""" + + @pytest.fixture(scope="class", autouse=True) + def setup_k8s(self): + """Setup Kubernetes client for tests.""" + try: + k8s_config.load_kube_config() + except Exception: + pytest.skip("No Kubernetes cluster available") + + @pytest.fixture(scope="class") + def test_namespace(self): + """Create test namespace for integration tests.""" + v1 = client.CoreV1Api() + namespace_name = "nio-integration-tests" + + # Create namespace + namespace = client.V1Namespace( + metadata=client.V1ObjectMeta(name=namespace_name) + ) + try: + v1.create_namespace(namespace) + except ApiException as e: + if e.status != 409: # Already exists + raise + + yield namespace_name + + # Cleanup: Delete namespace + try: + v1.delete_namespace(namespace_name) + except ApiException: + pass # Ignore cleanup errors + + @pytest.mark.skipif( + True, reason="Requires CRDs to be installed" + ) # Skip by default + def test_create_machine_resource(self, test_namespace): + """Should be able to create a Machine custom resource.""" + api = client.CustomObjectsApi() + + machine = { + "apiVersion": "nio.homystack.com/v1alpha1", + "kind": "Machine", + "metadata": { + "name": "test-machine", + "namespace": test_namespace, + }, + "spec": { + "hostname": "test.example.com", + "username": "root", + "credentialsRef": {"name": "test-creds"}, + }, + } + + try: + api.create_namespaced_custom_object( + group="nio.homystack.com", + version="v1alpha1", + namespace=test_namespace, + plural="machines", + body=machine, + ) + except ApiException as e: + # CRD might not be installed + if e.status == 404: + pytest.skip("Machine CRD not installed") + raise + + # Verify creation + created = api.get_namespaced_custom_object( + group="nio.homystack.com", + version="v1alpha1", + namespace=test_namespace, + plural="machines", + name="test-machine", + ) + assert created["metadata"]["name"] == "test-machine" + + # Cleanup + api.delete_namespaced_custom_object( + group="nio.homystack.com", + version="v1alpha1", + namespace=test_namespace, + plural="machines", + name="test-machine", + ) + + +class TestNixOSConfigurationResourceOperations: + """Integration tests for NixOSConfiguration custom resource operations.""" + + @pytest.fixture(scope="class", autouse=True) + def setup_k8s(self): + """Setup Kubernetes client for tests.""" + try: + k8s_config.load_kube_config() + except Exception: + pytest.skip("No Kubernetes cluster available") + + @pytest.fixture(scope="class") + def test_namespace(self): + """Create test namespace for integration tests.""" + v1 = client.CoreV1Api() + namespace_name = "nio-integration-tests" + + # Create namespace + namespace = client.V1Namespace( + metadata=client.V1ObjectMeta(name=namespace_name) + ) + try: + v1.create_namespace(namespace) + except ApiException as e: + if e.status != 409: # Already exists + raise + + yield namespace_name + + # Cleanup + try: + v1.delete_namespace(namespace_name) + except ApiException: + pass + + @pytest.mark.skipif( + True, reason="Requires CRDs to be installed" + ) # Skip by default + def test_create_nixos_configuration(self, test_namespace): + """Should be able to create a NixOSConfiguration custom resource.""" + api = client.CustomObjectsApi() + + config = { + "apiVersion": "nio.homystack.com/v1alpha1", + "kind": "NixOSConfiguration", + "metadata": { + "name": "test-config", + "namespace": test_namespace, + }, + "spec": { + "machineRef": {"name": "test-machine"}, + "gitRepo": "https://github.com/example/nixos-config.git", + "flakePath": ".#hostname", + }, + } + + try: + api.create_namespaced_custom_object( + group="nio.homystack.com", + version="v1alpha1", + namespace=test_namespace, + plural="nixosconfigurations", + body=config, + ) + except ApiException as e: + if e.status == 404: + pytest.skip("NixOSConfiguration CRD not installed") + raise + + # Verify creation + created = api.get_namespaced_custom_object( + group="nio.homystack.com", + version="v1alpha1", + namespace=test_namespace, + plural="nixosconfigurations", + name="test-config", + ) + assert created["metadata"]["name"] == "test-config" + + # Cleanup + api.delete_namespaced_custom_object( + group="nio.homystack.com", + version="v1alpha1", + namespace=test_namespace, + plural="nixosconfigurations", + name="test-config", + ) + + +class TestSecretOperations: + """Integration tests for Secret operations.""" + + @pytest.fixture(scope="class", autouse=True) + def setup_k8s(self): + """Setup Kubernetes client for tests.""" + try: + k8s_config.load_kube_config() + except Exception: + pytest.skip("No Kubernetes cluster available") + + @pytest.fixture(scope="class") + def test_namespace(self): + """Create test namespace for integration tests.""" + v1 = client.CoreV1Api() + namespace_name = "nio-integration-tests" + + namespace = client.V1Namespace( + metadata=client.V1ObjectMeta(name=namespace_name) + ) + try: + v1.create_namespace(namespace) + except ApiException as e: + if e.status != 409: + raise + + yield namespace_name + + try: + v1.delete_namespace(namespace_name) + except ApiException: + pass + + def test_create_and_read_secret(self, test_namespace): + """Should be able to create and read secrets.""" + v1 = client.CoreV1Api() + + secret = client.V1Secret( + metadata=client.V1ObjectMeta(name="test-secret"), + string_data={"ssh-privatekey": "test-key-content"}, + ) + + # Create secret + v1.create_namespaced_secret(test_namespace, secret) + + # Read secret + read_secret = v1.read_namespaced_secret("test-secret", test_namespace) + assert read_secret.metadata.name == "test-secret" + assert "ssh-privatekey" in read_secret.data + + # Cleanup + v1.delete_namespaced_secret("test-secret", test_namespace) diff --git a/tests/test_known_hosts_manager.py b/tests/test_known_hosts_manager.py new file mode 100644 index 0000000..cecfac6 --- /dev/null +++ b/tests/test_known_hosts_manager.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 + +"""Unit tests for SSH known_hosts manager module.""" + +import pytest +import tempfile +import os +from pathlib import Path +from unittest.mock import patch, MagicMock +from known_hosts_manager import KnownHostsManager, get_known_hosts_manager + + +class TestKnownHostsManagerInitialization: + """Tests for KnownHostsManager initialization.""" + + def test_init_with_custom_storage_path(self): + """Should initialize with custom storage path.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "custom_known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + assert manager.known_hosts_path == Path(storage_path) + assert manager.known_hosts_path.exists() + # Check file permissions (0o600 = rw-------) + assert oct(manager.known_hosts_path.stat().st_mode)[-3:] == "600" + + def test_init_with_default_storage_path(self): + """Should initialize with default storage path from config.""" + with patch("known_hosts_manager.config.KNOWN_HOSTS_PATH", "/tmp/test-known-hosts"): + manager = KnownHostsManager() + + assert manager.known_hosts_path == Path("/tmp/test-known-hosts/known_hosts") + assert manager.known_hosts_path.exists() + + def test_get_known_hosts_path(self): + """Should return known_hosts file path as string.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + path = manager.get_known_hosts_path() + + assert isinstance(path, str) + assert path == str(manager.known_hosts_path) + + +class TestKnownHostsManagerAddHostKey: + """Tests for adding host keys.""" + + def test_add_new_host_key(self): + """Should add new host key to known_hosts.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + hostname = "192.168.1.100" + key_type = "ssh-ed25519" + public_key = "AAAAC3NzaC1lZDI1NTE5AAAAIFakeKeyForTesting123456" + + manager.add_host_key(hostname, key_type, public_key) + + with open(manager.known_hosts_path, "r") as f: + content = f.read() + assert f"{hostname} {key_type} {public_key}" in content + + def test_add_duplicate_host_key(self): + """Should not add duplicate host key.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + hostname = "192.168.1.100" + key_type = "ssh-ed25519" + public_key = "AAAAC3NzaC1lZDI1NTE5AAAAIFakeKeyForTesting123456" + + # Add key twice + manager.add_host_key(hostname, key_type, public_key) + manager.add_host_key(hostname, key_type, public_key) + + with open(manager.known_hosts_path, "r") as f: + content = f.read() + # Should appear only once + assert content.count(f"{hostname} {key_type} {public_key}") == 1 + + def test_add_multiple_different_hosts(self): + """Should add multiple different host keys.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + hosts = [ + ("host1.example.com", "ssh-ed25519", "AAAAC3NzaC1lZDI1NTE5AAAAIKey1"), + ("host2.example.com", "ecdsa-sha2-nistp256", "AAAAE2VjZHNhKey2"), + ("192.168.1.100", "ssh-rsa", "AAAAB3NzaC1yc2EAAAAKey3"), + ] + + for hostname, key_type, public_key in hosts: + manager.add_host_key(hostname, key_type, public_key) + + with open(manager.known_hosts_path, "r") as f: + content = f.read() + for hostname, key_type, public_key in hosts: + assert f"{hostname} {key_type} {public_key}" in content + + +class TestKnownHostsManagerTOFU: + """Tests for Trust On First Use functionality.""" + + def test_tofu_first_connection(self): + """Should return True for first connection (new host).""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + result = manager.trust_on_first_use("new-host.example.com") + + assert result is True + + def test_tofu_subsequent_connection_standard_port(self): + """Should return False for known host on standard port.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + hostname = "known-host.example.com" + manager.add_host_key(hostname, "ssh-ed25519", "FakePublicKey123") + + result = manager.trust_on_first_use(hostname) + + assert result is False + + def test_tofu_subsequent_connection_custom_port(self): + """Should return False for known host on custom port.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + hostname = "192.168.1.100" + port = 2222 + # Add host with port notation [hostname]:port + with open(manager.known_hosts_path, "a") as f: + f.write(f"[{hostname}]:{port} ssh-ed25519 FakeKey\n") + + result = manager.trust_on_first_use(hostname, port) + + assert result is False + + def test_tofu_nonexistent_known_hosts_file(self): + """Should return True when known_hosts file doesn't exist.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + # Delete the file that was created during init + manager.known_hosts_path.unlink() + + result = manager.trust_on_first_use("any-host.example.com") + + assert result is True + + +class TestKnownHostsManagerClearHost: + """Tests for clearing host keys.""" + + def test_clear_existing_host(self): + """Should remove all entries for specified host.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + # Add multiple hosts + manager.add_host_key("host1.example.com", "ssh-ed25519", "Key1") + manager.add_host_key("host2.example.com", "ssh-ed25519", "Key2") + manager.add_host_key("host3.example.com", "ssh-ed25519", "Key3") + + manager.clear_host("host2.example.com") + + with open(manager.known_hosts_path, "r") as f: + content = f.read() + assert "host1.example.com" in content + assert "host2.example.com" not in content + assert "host3.example.com" in content + + def test_clear_host_with_port_notation(self): + """Should remove host with port notation [hostname]:port.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + hostname = "192.168.1.100" + # Add host with port notation + with open(manager.known_hosts_path, "a") as f: + f.write(f"[{hostname}]:2222 ssh-ed25519 FakeKey\n") + + manager.clear_host(hostname) + + with open(manager.known_hosts_path, "r") as f: + content = f.read() + assert hostname not in content + + def test_clear_nonexistent_host(self): + """Should handle clearing nonexistent host gracefully.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + manager.add_host_key("host1.example.com", "ssh-ed25519", "Key1") + + # Clear nonexistent host - should not raise error + manager.clear_host("nonexistent.example.com") + + with open(manager.known_hosts_path, "r") as f: + content = f.read() + assert "host1.example.com" in content + + def test_clear_host_when_file_does_not_exist(self): + """Should handle clearing when known_hosts file doesn't exist.""" + with tempfile.TemporaryDirectory() as temp_dir: + storage_path = os.path.join(temp_dir, "known_hosts") + manager = KnownHostsManager(storage_path=storage_path) + + # Delete the file + manager.known_hosts_path.unlink() + + # Should not raise error + manager.clear_host("any-host.example.com") + + +class TestGlobalKnownHostsManager: + """Tests for global singleton manager.""" + + def test_get_known_hosts_manager_singleton(self): + """Should return same instance on multiple calls.""" + # Reset global state + import known_hosts_manager + known_hosts_manager._known_hosts_manager = None + + manager1 = get_known_hosts_manager() + manager2 = get_known_hosts_manager() + + assert manager1 is manager2 + + def test_get_known_hosts_manager_creates_instance(self): + """Should create instance if none exists.""" + import known_hosts_manager + known_hosts_manager._known_hosts_manager = None + + manager = get_known_hosts_manager() + + assert isinstance(manager, KnownHostsManager) + assert manager is not None diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..ff94b58 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 + +"""Unit tests for Prometheus metrics module.""" + +import pytest +from prometheus_client import REGISTRY +from metrics import ( + init_metrics, + record_reconcile_success, + record_reconcile_error, + record_ssh_connection, + record_git_clone, + record_nixos_build, + operator_info, + machines_total, + reconcile_duration, + ssh_connections_total, + git_clones_total, + nixos_builds_total, +) + + +class TestMetricsInitialization: + """Tests for metrics initialization.""" + + def test_init_metrics(self): + """init_metrics should set operator info.""" + init_metrics() + # Verify operator_info was set (it's a Counter-like metric) + # We can't easily inspect Info metrics, but we can verify it doesn't crash + assert operator_info is not None + + +class TestMetricsRecording: + """Tests for metric recording helper functions.""" + + def test_record_reconcile_success(self): + """Should record successful reconciliation with duration.""" + namespace = "test-ns" + configuration = "test-config" + duration = 1.5 + + # Record before value + before = reconcile_duration.labels( + namespace=namespace, configuration=configuration + )._sum.get() + + record_reconcile_success(namespace, configuration, duration) + + # Record after value + after = reconcile_duration.labels( + namespace=namespace, configuration=configuration + )._sum.get() + + # Duration should increase + assert after > before + + def test_record_reconcile_error(self): + """Should increment reconciliation error counter.""" + namespace = "test-ns" + configuration = "test-config" + error_type = "GitCloneError" + + # Get before value + metric = REGISTRY.get_sample_value( + "nio_reconcile_errors_total", + labels={ + "namespace": namespace, + "configuration": configuration, + "error_type": error_type, + }, + ) + before = metric if metric is not None else 0 + + record_reconcile_error(namespace, configuration, error_type) + + # Get after value + metric = REGISTRY.get_sample_value( + "nio_reconcile_errors_total", + labels={ + "namespace": namespace, + "configuration": configuration, + "error_type": error_type, + }, + ) + after = metric if metric is not None else 0 + + assert after == before + 1 + + def test_record_ssh_connection_success(self): + """Should record successful SSH connection.""" + namespace = "test-ns" + machine = "test-machine" + duration = 0.5 + + # Get before value + metric = REGISTRY.get_sample_value( + "nio_ssh_connections_total", + labels={"namespace": namespace, "machine": machine, "result": "success"}, + ) + before = metric if metric is not None else 0 + + record_ssh_connection(namespace, machine, success=True, duration=duration) + + # Get after value + metric = REGISTRY.get_sample_value( + "nio_ssh_connections_total", + labels={"namespace": namespace, "machine": machine, "result": "success"}, + ) + after = metric if metric is not None else 0 + + assert after == before + 1 + + def test_record_ssh_connection_failure(self): + """Should record failed SSH connection.""" + namespace = "test-ns" + machine = "test-machine" + duration = 0.1 + + # Get before value + metric = REGISTRY.get_sample_value( + "nio_ssh_connections_total", + labels={"namespace": namespace, "machine": machine, "result": "failure"}, + ) + before = metric if metric is not None else 0 + + record_ssh_connection(namespace, machine, success=False, duration=duration) + + # Get after value + metric = REGISTRY.get_sample_value( + "nio_ssh_connections_total", + labels={"namespace": namespace, "machine": machine, "result": "failure"}, + ) + after = metric if metric is not None else 0 + + assert after == before + 1 + + def test_record_git_clone_success(self): + """Should record successful git clone.""" + namespace = "test-ns" + repository = "owner/repo" + duration = 2.0 + + # Get before value + metric = REGISTRY.get_sample_value( + "nio_git_clones_total", + labels={ + "namespace": namespace, + "repository": repository, + "result": "success", + }, + ) + before = metric if metric is not None else 0 + + record_git_clone(namespace, repository, success=True, duration=duration) + + # Get after value + metric = REGISTRY.get_sample_value( + "nio_git_clones_total", + labels={ + "namespace": namespace, + "repository": repository, + "result": "success", + }, + ) + after = metric if metric is not None else 0 + + assert after == before + 1 + + def test_record_git_clone_failure(self): + """Should record failed git clone.""" + namespace = "test-ns" + repository = "owner/repo" + duration = 0.5 + + # Get before value + metric = REGISTRY.get_sample_value( + "nio_git_clones_total", + labels={ + "namespace": namespace, + "repository": repository, + "result": "failure", + }, + ) + before = metric if metric is not None else 0 + + record_git_clone(namespace, repository, success=False, duration=duration) + + # Get after value + metric = REGISTRY.get_sample_value( + "nio_git_clones_total", + labels={ + "namespace": namespace, + "repository": repository, + "result": "failure", + }, + ) + after = metric if metric is not None else 0 + + assert after == before + 1 + + def test_record_nixos_build_success(self): + """Should record successful NixOS build.""" + namespace = "test-ns" + machine = "test-machine" + build_type = "switch" + duration = 300.0 + + # Get before value + metric = REGISTRY.get_sample_value( + "nio_nixos_builds_total", + labels={ + "namespace": namespace, + "machine": machine, + "build_type": build_type, + "result": "success", + }, + ) + before = metric if metric is not None else 0 + + record_nixos_build(namespace, machine, build_type, success=True, duration=duration) + + # Get after value + metric = REGISTRY.get_sample_value( + "nio_nixos_builds_total", + labels={ + "namespace": namespace, + "machine": machine, + "build_type": build_type, + "result": "success", + }, + ) + after = metric if metric is not None else 0 + + assert after == before + 1 + + def test_record_nixos_build_failure(self): + """Should record failed NixOS build.""" + namespace = "test-ns" + machine = "test-machine" + build_type = "boot" + duration = 10.0 + + # Get before value + metric = REGISTRY.get_sample_value( + "nio_nixos_builds_total", + labels={ + "namespace": namespace, + "machine": machine, + "build_type": build_type, + "result": "failure", + }, + ) + before = metric if metric is not None else 0 + + record_nixos_build(namespace, machine, build_type, success=False, duration=duration) + + # Get after value + metric = REGISTRY.get_sample_value( + "nio_nixos_builds_total", + labels={ + "namespace": namespace, + "machine": machine, + "build_type": build_type, + "result": "failure", + }, + ) + after = metric if metric is not None else 0 + + assert after == before + 1 + + +class TestMetricsAvailability: + """Tests that all expected metrics are defined.""" + + def test_all_metrics_exist(self): + """All documented metrics should be defined.""" + # Just verify they're importable and have expected types + assert operator_info is not None + assert machines_total is not None + assert reconcile_duration is not None + assert ssh_connections_total is not None + assert git_clones_total is not None + assert nixos_builds_total is not None diff --git a/tests/test_retry_utils.py b/tests/test_retry_utils.py new file mode 100644 index 0000000..4d5540c --- /dev/null +++ b/tests/test_retry_utils.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 + +"""Unit tests for retry utilities.""" + +import pytest +import asyncio +from retry_utils import ( + retry_with_backoff, + with_retry, + RetryableOperation, + RetryExhaustedError, +) + + +class TestRetryWithBackoff: + """Tests for retry_with_backoff function.""" + + @pytest.mark.asyncio + async def test_succeeds_first_try(self): + """Function that succeeds on first try should not retry.""" + call_count = 0 + + async def succeeds_immediately(): + nonlocal call_count + call_count += 1 + return "success" + + result = await retry_with_backoff(succeeds_immediately, max_attempts=3) + + assert result == "success" + assert call_count == 1 # Only called once + + @pytest.mark.asyncio + async def test_succeeds_after_retries(self): + """Function that fails then succeeds should retry.""" + call_count = 0 + + async def fails_then_succeeds(): + nonlocal call_count + call_count += 1 + if call_count < 3: + raise Exception("Temporary failure") + return "success" + + result = await retry_with_backoff( + fails_then_succeeds, max_attempts=5, initial_delay=0.01 + ) + + assert result == "success" + assert call_count == 3 # Failed twice, succeeded third time + + @pytest.mark.asyncio + async def test_exhausts_retries(self): + """Function that always fails should exhaust retries.""" + call_count = 0 + + async def always_fails(): + nonlocal call_count + call_count += 1 + raise Exception("Permanent failure") + + with pytest.raises(RetryExhaustedError, match="failed after 3 attempts"): + await retry_with_backoff( + always_fails, max_attempts=3, initial_delay=0.01 + ) + + assert call_count == 3 # Tried max_attempts times + + @pytest.mark.asyncio + async def test_exponential_backoff(self): + """Delay should increase exponentially.""" + delays = [] + call_count = 0 + + async def track_delays(): + nonlocal call_count + call_count += 1 + if call_count > 1: + # Record time since start would be better, but this is simpler + delays.append(call_count) + raise Exception("Failure") + + with pytest.raises(Exception): + await retry_with_backoff( + track_delays, + max_attempts=4, + initial_delay=0.01, + exponential_base=2.0, + jitter=False, + ) + + # Should have tried 4 times + assert call_count == 4 + + +class TestWithRetryDecorator: + """Tests for with_retry decorator.""" + + @pytest.mark.asyncio + async def test_decorator_succeeds(self): + """Decorated function should work normally.""" + call_count = 0 + + @with_retry(max_attempts=3, initial_delay=0.01) + async def decorated_function(): + nonlocal call_count + call_count += 1 + return "success" + + result = await decorated_function() + + assert result == "success" + assert call_count == 1 + + @pytest.mark.asyncio + async def test_decorator_retries(self): + """Decorated function should retry on failure.""" + call_count = 0 + + @with_retry(max_attempts=3, initial_delay=0.01) + async def decorated_function(): + nonlocal call_count + call_count += 1 + if call_count < 2: + raise Exception("Temporary failure") + return "success" + + result = await decorated_function() + + assert result == "success" + assert call_count == 2 # Failed once, succeeded second time + + +class TestRetryableOperation: + """Tests for RetryableOperation context manager.""" + + @pytest.mark.asyncio + async def test_context_manager_succeeds(self): + """Context manager should work for successful operations.""" + call_count = 0 + + async def do_operation(): + nonlocal call_count + call_count += 1 + return "success" + + async with RetryableOperation("test_op", max_attempts=3) as op: + result = await do_operation() + + assert result == "success" + assert call_count == 1 + + @pytest.mark.asyncio + async def test_context_manager_with_manual_retry(self): + """Context manager should support manual retry in a loop.""" + call_count = 0 + + async def do_operation(): + nonlocal call_count + call_count += 1 + if call_count < 2: + raise Exception("Temporary failure") + return "success" + + op = RetryableOperation("test_op", max_attempts=3, initial_delay=0.01) + result = None + + for _ in range(op.max_attempts): + try: + async with op: + result = await do_operation() + break # Success + except Exception: + if op.attempt >= op.max_attempts: + raise + + assert result == "success" + assert call_count == 2 # Failed once, succeeded second time diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..c1e9329 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +"""Unit tests for utility functions.""" + +import pytest +import tempfile +import os +import shutil +from utils import ( + extract_repo_name_from_url, + calculate_directory_hash, + get_workdir_path, + parse_flake_reference, +) + + +class TestExtractRepoNameFromUrl: + """Tests for repository name extraction.""" + + def test_https_url(self): + """Extract repo name from HTTPS URL.""" + url = "https://github.com/owner/repo.git" + result = extract_repo_name_from_url(url) + assert result == "owner/repo" + + def test_https_url_without_git(self): + """Extract repo name from HTTPS URL without .git.""" + url = "https://github.com/owner/repo" + result = extract_repo_name_from_url(url) + assert result == "owner/repo" + + def test_ssh_url(self): + """Extract repo name from SSH URL.""" + url = "git@github.com:owner/repo.git" + result = extract_repo_name_from_url(url) + assert result == "owner/repo" + + +class TestCalculateDirectoryHash: + """Tests for directory hash calculation.""" + + def test_empty_directory(self): + """Hash of empty directory should be empty string.""" + with tempfile.TemporaryDirectory() as tmpdir: + result = calculate_directory_hash(tmpdir) + # Empty directory should produce a hash (SHA256 of empty data) + assert isinstance(result, str) + assert len(result) == 64 # SHA256 hex length + + def test_directory_with_files(self): + """Hash should be deterministic for same content.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create some files + with open(os.path.join(tmpdir, "file1.txt"), "w") as f: + f.write("content1") + with open(os.path.join(tmpdir, "file2.txt"), "w") as f: + f.write("content2") + + hash1 = calculate_directory_hash(tmpdir) + hash2 = calculate_directory_hash(tmpdir) + + # Hash should be deterministic + assert hash1 == hash2 + assert len(hash1) == 64 # SHA256 hex length + + def test_different_content_different_hash(self): + """Different content should produce different hash.""" + with tempfile.TemporaryDirectory() as tmpdir1: + with tempfile.TemporaryDirectory() as tmpdir2: + # Create different files in each + with open(os.path.join(tmpdir1, "file.txt"), "w") as f: + f.write("content1") + with open(os.path.join(tmpdir2, "file.txt"), "w") as f: + f.write("content2") + + hash1 = calculate_directory_hash(tmpdir1) + hash2 = calculate_directory_hash(tmpdir2) + + # Hashes should be different + assert hash1 != hash2 + + def test_nonexistent_directory(self): + """Nonexistent directory should return empty string.""" + result = calculate_directory_hash("/nonexistent/path") + assert result == "" + + +class TestGetWorkdirPath: + """Tests for working directory path generation.""" + + def test_creates_directory(self): + """get_workdir_path should create directory if it doesn't exist.""" + with tempfile.TemporaryDirectory() as base: + # Temporarily override BASE_CONFIG_PATH + import config + original = config.BASE_CONFIG_PATH + try: + config.BASE_CONFIG_PATH = base + path = get_workdir_path("test-ns", "test-config", "owner/repo", "abc123") + + # Path should be created + assert os.path.exists(path) + assert "test-ns" in path + assert "test-config" in path + assert "owner/repo" in path + assert "abc123" in path + finally: + config.BASE_CONFIG_PATH = original + + def test_path_format(self): + """Working directory path should follow expected format.""" + with tempfile.TemporaryDirectory() as base: + import config + original = config.BASE_CONFIG_PATH + try: + config.BASE_CONFIG_PATH = base + path = get_workdir_path("ns", "name", "owner/repo", "commit") + + # Should contain all components + assert base in path + assert path.endswith("owner/repo@commit") + finally: + config.BASE_CONFIG_PATH = original + + +class TestParseFlakeReference: + """Tests for flake reference parsing.""" + + def test_github_flake(self): + """Parse github: flake reference.""" + ref = "github:owner/repo#hostname" + repo_name, repo_url, commit = parse_flake_reference(ref) + + assert repo_name == "owner/repo" + assert repo_url == "https://github.com/owner/repo.git" + assert commit == "floating" # No specific commit + + def test_github_flake_with_ref(self): + """Parse github: flake with branch/tag reference.""" + ref = "github:owner/repo/v1.0#hostname" + repo_name, repo_url, commit = parse_flake_reference(ref) + + assert repo_name == "owner/repo" + assert repo_url == "https://github.com/owner/repo.git" + assert commit == "floating" # Branch/tag, not commit + + def test_github_flake_with_commit(self): + """Parse github: flake with commit hash.""" + commit_hash = "a" * 40 # 40-char commit hash + ref = f"github:owner/repo/{commit_hash}#hostname" + repo_name, repo_url, commit = parse_flake_reference(ref) + + assert repo_name == "owner/repo" + assert repo_url == "https://github.com/owner/repo.git" + assert commit == commit_hash + + def test_local_flake(self): + """Parse local flake reference.""" + ref = ".#hostname" + repo_name, repo_url, commit = parse_flake_reference(ref) + + assert repo_name == "local" + assert repo_url == "." + assert commit == "local" + + def test_unknown_flake_source(self): + """Parse unknown flake source.""" + ref = "unknown:some/path#hostname" + repo_name, repo_url, commit = parse_flake_reference(ref) + + assert repo_name == "unknown" + assert repo_url == "unknown:some/path" + assert commit == "unknown" + + +class TestExtractRepoNameEdgeCases: + """Tests for repository name extraction edge cases.""" + + def test_http_url(self): + """Extract from HTTP URL (not HTTPS).""" + url = "http://gitlab.com/owner/repo.git" + result = extract_repo_name_from_url(url) + assert result == "owner/repo" + + def test_ssh_url_without_git_suffix(self): + """Extract from SSH URL without .git suffix.""" + url = "git@github.com:owner/repo" + result = extract_repo_name_from_url(url) + assert result == "owner/repo" + + def test_url_with_subdirectories(self): + """Extract from URL with more than 2 path components.""" + url = "https://example.com/group/subgroup/owner/repo.git" + result = extract_repo_name_from_url(url) + # Should extract last two components + assert result == "owner/repo" diff --git a/utils.py b/utils.py index 38cdc5b..31630b0 100644 --- a/utils.py +++ b/utils.py @@ -7,18 +7,19 @@ import os import re import hashlib -from typing import Dict, Optional, Tuple, List +from typing import Dict, Optional, Tuple, List, Any from datetime import datetime from clients import get_secret_data +from input_validation import validate_git_url, ValidationError +import config def get_workdir_path( namespace: str, name: str, repo_name: str, commit_hash: str ) -> str: """Get predictable working directory path""" - base_path = "/tmp/nixos-config" - workdir = f"{base_path}/{namespace}/{name}/{repo_name}@{commit_hash}" + workdir = f"{config.BASE_CONFIG_PATH}/{namespace}/{name}/{repo_name}@{commit_hash}" os.makedirs(workdir, exist_ok=True) return workdir @@ -71,6 +72,13 @@ def parse_flake_reference(flake_ref: str) -> Tuple[str, str, str]: def extract_repo_name_from_url(git_url: str) -> str: """Extract repository name from Git URL""" + # Handle SSH URLs (git@github.com:owner/repo.git) + if git_url.startswith("git@"): + # Remove git@ prefix and .git suffix + clean_url = re.sub(r"^git@[^:]+:", "", git_url) + clean_url = re.sub(r"\.git$", "", clean_url) + return clean_url + # Remove protocol and .git clean_url = re.sub(r"^https?://", "", git_url) clean_url = re.sub(r"\.git$", "", clean_url) @@ -120,6 +128,12 @@ async def clone_git_repo( target_path: Optional[str] = None, ) -> Tuple[str, str]: """Clone Git repository and return path and commit hash""" + # SECURITY: Validate Git URL to prevent command injection + try: + git_url = validate_git_url(git_url) + except ValidationError as e: + raise ValueError(f"Invalid Git URL: {e}") + if target_path: work_dir = target_path # If directory already exists, use it