diff --git a/.gitignore b/.gitignore index 855c296..2a14aea 100644 --- a/.gitignore +++ b/.gitignore @@ -26,8 +26,6 @@ coverage/ .tmp/ # generated by hyperi-ci's osv-scanner stage from quality.ignore (mirrors deny.toml) osv-scanner.toml -# written by `dfe-loader --emit-helm`; the release assembles the chart from the contract -/chart/ # Rust / cargo build artefacts /target diff --git a/.hyperi-ci.yaml b/.hyperi-ci.yaml index 772eff4..bd3b2c8 100644 --- a/.hyperi-ci.yaml +++ b/.hyperi-ci.yaml @@ -71,7 +71,7 @@ release: helm: enabled: true contract: emit - library: "2.14.3" + library: "2.14.6" container: enabled: true dockerfile: Dockerfile diff --git a/Cargo.lock b/Cargo.lock index 7a5c998..eec3fcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3555,7 +3555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -3576,7 +3576,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", @@ -3665,7 +3665,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.5", + "socket2 0.5.10", "thiserror 2.0.21", "tokio", "tracing", @@ -3704,7 +3704,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.5", + "socket2 0.5.10", "tracing", "windows-sys 0.61.2", ] @@ -4373,9 +4373,9 @@ dependencies = [ [[package]] name = "scalo" -version = "2.14.3" +version = "2.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1105393fd98b1d95b1809650b5761794a7df88a21a72dddce4ca535d946e983f" +checksum = "c9e521cd4d8b8f5c4acf17cdecacbbf25e7688e812d008439ae09f12f34b43ab" dependencies = [ "aes-gcm", "backon", diff --git a/Cargo.toml b/Cargo.toml index 338f8f6..1ede6d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,9 +31,10 @@ required-features = ["pgo-driver"] tokio = { version = ">=1.53.2, <2", features = ["full"] } tokio-util = { version = ">=0.7.19, <0.8", features = ["rt"] } -# Shared library with transport abstraction. The floor is the first release whose -# contract carries no vendor defaults, so the app names its own. -scalo = { version = "2.14.3", features = ["config-schema", "transport-kafka", "transport-grpc", "dlq-kafka", "config", "config-reload", "deployment", "version-check", "scaling", "cli-service", "top", "logger", "metrics", "service-metrics", "expression", "memory", "worker-batch", "governor", "secrets", "sink-stack"] } +# Shared library with transport abstraction. The floor moves with +# `release.helm.library` in .hyperi-ci.yaml: the library renders only the +# contract version its scalo release writes. +scalo = { version = "2.14.6", features = ["config-schema", "transport-kafka", "transport-grpc", "dlq-kafka", "config", "config-reload", "deployment", "version-check", "scaling", "cli-service", "top", "logger", "metrics", "service-metrics", "expression", "memory", "worker-batch", "governor", "secrets", "sink-stack"] } # Steps scalo's jittered retry schedule (sink_stack backoff). Range MUST track # scalo's: BackoffBuilder only applies to the ExponentialBuilder scalo returns. diff --git a/README.md b/README.md index 1f959f2..8b4b86f 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Inbound, declared in `dfe-infra/suite.yaml`: - **scalo-rs -> dfe-loader** (`generated-file`, lockstep) -- `Dockerfile` is emitted by `scalo::deployment::generate_dockerfile()` at schema version 4. Regenerate and commit the diff. Outbound, so what a change here can break: -- **dfe-loader -> dfe-infra** (`image-pin`, lockstep) -- `dfe-infra/helm/charts/dfe-loader/Chart.yaml:6` pins this image as tag plus digest. +- **dfe-loader -> dfe-infra** (`image-pin`, lockstep) -- `dfe-infra/versions.yaml` pins this image and the thin chart released here, and Argo deploys that published chart. - **dfe-loader -> dfe-engine** (`mirrored-logic`) -- dfe-engine hand-reimplements this repo's config validation in `plugins_builtin/loader.py` to catch errors when it authors a config. Nothing is copied, so no script compares them. Runtime, not build edges: dfe-engine's `dfe-schema` entry point creates every destination table before the loader can insert, and producers reach the loader over Kafka (`*_land` / `*_load` topics) or direct gRPC (default topic `main_land`) -- in the suite that producer is dfe-receiver. diff --git a/config/defaults.yaml b/config/defaults.yaml deleted file mode 100644 index 8d13879..0000000 --- a/config/defaults.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Project: dfe-loader -# File: config/defaults.yaml -# Purpose: Cascade defaults for `dfe-loader generate-artefacts` -# -# License: BUSL-1.1 -# Copyright: (c) 2026 HYPERI PTY LIMITED - -# generate-artefacts writes no ArgoCD Application without a repo URL, and -# deploys into a namespace named after the app unless one is set here. -deployment: - argocd: - repo_url: https://github.com/hyperi-io/dfe-loader - dest_namespace: dfe diff --git a/src/config/loader.rs b/src/config/loader.rs index 94d43c7..2a1e62e 100644 --- a/src/config/loader.rs +++ b/src/config/loader.rs @@ -575,13 +575,14 @@ impl Config { /// Build a deployment contract from loader config defaults. /// /// Apps provide ~20% customisation; scalo generates ~80% boilerplate - /// (Dockerfile, Helm chart, Compose fragment). + /// (Dockerfile, Compose fragment), and the release assembles the thin + /// chart from this contract. pub fn deployment_contract() -> scalo::deployment::DeploymentContract { use scalo::deployment::{ CONTRACT_SCHEMA_VERSION, DeploymentContract, HealthContract, ImageProfile, NativeDepsContract, OciLabels, PortCondition, PortContract, ResourceList, - ResourcesContract, SecurityContract, WritablePath, base_image_from_cascade, - image_registry_from_cascade, + ResourcesContract, SecurityContract, ServiceAccount, WritablePath, + base_image_from_cascade, image_registry_from_cascade, }; // A deployment.* cascade key (or env) wins over the base image and registry @@ -683,6 +684,7 @@ impl Config { }, security: SecurityContract::default(), singleton: false, + service_account: ServiceAccount::Own, } } diff --git a/src/main.rs b/src/main.rs index 0965058..5877284 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,10 +39,6 @@ struct App { #[command(subcommand)] command: Option, - /// Generate Helm chart from deployment contract and exit (default output: ./chart) - #[arg(long, value_name = "DIR", default_missing_value = "chart", num_args = 0..=1)] - emit_helm: Option, - /// Generate Dockerfile from deployment contract and exit (default output: ./Dockerfile) #[arg(long, value_name = "FILE", default_missing_value = "Dockerfile", num_args = 0..=1)] emit_dockerfile: Option, @@ -315,16 +311,6 @@ async fn main() { let app = App::parse(); - if let Some(output) = &app.emit_helm { - let contract = Config::deployment_contract(); - if let Err(e) = scalo::deployment::generate_chart(&contract, output, None) { - eprintln!("fatal: {e}"); - std::process::exit(1); - } - println!("Helm chart generated at {}", output.display()); - return; - } - if let Some(output) = &app.emit_dockerfile { let contract = Config::deployment_contract(); let content = scalo::deployment::generate_dockerfile(&contract, None); diff --git a/tests/integration/deployment.rs b/tests/integration/deployment.rs index 168c357..a7b4634 100644 --- a/tests/integration/deployment.rs +++ b/tests/integration/deployment.rs @@ -3,15 +3,13 @@ //! Deployment artifact generation tests //! -//! Verifies that generated Dockerfile and Helm chart match the contract values. +//! Verifies that the generated Dockerfile and compose fragment match the +//! contract values. use std::path::Path; use dfe_loader::config::Config; -use scalo::deployment::{ - generate_chart, generate_compose_fragment, generate_dockerfile, validate_dockerfile, - validate_helm_values, -}; +use scalo::deployment::{generate_compose_fragment, generate_dockerfile, validate_dockerfile}; fn contract() -> scalo::deployment::DeploymentContract { Config::deployment_contract() @@ -53,89 +51,26 @@ fn test_generated_dockerfile_matches_existing() { ); } +/// The library chart injects each secret env var the contract declares, and +/// `Config::load` reads exactly these names. #[test] -fn test_generated_chart_structure() { +fn test_contract_declares_secret_env_vars() { let contract = contract(); - let dir = tempfile::tempdir().unwrap(); - generate_chart(&contract, dir.path(), None).unwrap(); - - // Chart.yaml - let chart = std::fs::read_to_string(dir.path().join("Chart.yaml")).unwrap(); - assert!(chart.contains("name: dfe-loader")); - assert!(chart.contains("description: High-performance Kafka to ClickHouse data loader")); - - // values.yaml - let values = std::fs::read_to_string(dir.path().join("values.yaml")).unwrap(); - assert!(values.contains("repository: ghcr.io/hyperi-io/dfe-loader")); - assert!(values.contains("port: 9090")); - - // Secret sections - assert!( - values.contains("kafka:"), - "Should have kafka secret section" - ); - assert!( - values.contains("clickhouse:"), - "Should have clickhouse secret section" - ); - assert!( - values.contains("existingSecret:"), - "Should support existing secrets" - ); - - // KEDA section - assert!(values.contains("keda:"), "Should have KEDA section"); - - // templates/ - let templates = dir.path().join("templates"); - assert!(templates.join("_helpers.tpl").exists()); - assert!(templates.join("deployment.yaml").exists()); - assert!(templates.join("service.yaml").exists()); - assert!(templates.join("serviceaccount.yaml").exists()); - assert!(templates.join("configmap.yaml").exists()); - assert!(templates.join("secret.yaml").exists()); - assert!(templates.join("hpa.yaml").exists()); - assert!(templates.join("keda-scaledobject.yaml").exists()); - assert!(templates.join("keda-triggerauth.yaml").exists()); - assert!(templates.join("NOTES.txt").exists()); -} - -#[test] -fn test_generated_chart_helpers() { - let contract = contract(); - let dir = tempfile::tempdir().unwrap(); - generate_chart(&contract, dir.path(), None).unwrap(); - - let helpers = std::fs::read_to_string(dir.path().join("templates/_helpers.tpl")).unwrap(); - assert!( - helpers.contains("kafkaSecretName"), - "Should have kafka secret helper" - ); - assert!( - helpers.contains("clickhouseSecretName"), - "Should have clickhouse secret helper" - ); -} - -#[test] -fn test_generated_deployment_has_env_vars() { - let contract = contract(); - let dir = tempfile::tempdir().unwrap(); - generate_chart(&contract, dir.path(), None).unwrap(); - - let deploy = std::fs::read_to_string(dir.path().join("templates/deployment.yaml")).unwrap(); - assert!( - deploy.contains("DFE_LOADER__KAFKA__SASL__USERNAME"), - "Should inject kafka username env" - ); - assert!( - deploy.contains("DFE_LOADER__KAFKA__SASL__PASSWORD"), - "Should inject kafka password env" - ); - assert!( - deploy.contains("DFE_LOADER__CLICKHOUSE__PASSWORD"), - "Should inject clickhouse password env" - ); + let env_vars: Vec<&str> = contract + .secrets + .iter() + .flat_map(|group| group.env_vars.iter().map(|env| env.env_var.as_str())) + .collect(); + for name in [ + "DFE_LOADER__KAFKA__SASL__USERNAME", + "DFE_LOADER__KAFKA__SASL__PASSWORD", + "DFE_LOADER__CLICKHOUSE__PASSWORD", + ] { + assert!( + env_vars.contains(&name), + "contract should declare {name}: {env_vars:?}" + ); + } } #[test] @@ -168,20 +103,6 @@ fn test_validate_generated_dockerfile() { ); } -#[test] -fn test_validate_generated_chart() { - let contract = contract(); - let dir = tempfile::tempdir().unwrap(); - generate_chart(&contract, dir.path(), None).unwrap(); - - // Generated chart should pass validation against its own contract - let result = validate_helm_values(&contract, dir.path().to_str().unwrap()); - assert!( - result.is_ok(), - "Generated chart should validate: {result:?}" - ); -} - /// Write generated artifacts to .tmp/ for manual comparison. /// Run with: cargo test --test deployment -- `write_artifacts` --ignored --nocapture #[test] @@ -199,10 +120,5 @@ fn write_artifacts_to_tmp() { let compose = generate_compose_fragment(&contract); std::fs::write(base.join("compose.service.yaml"), &compose).unwrap(); - // Helm chart - let chart_dir = base.join("chart"); - let _ = std::fs::remove_dir_all(&chart_dir); - generate_chart(&contract, &chart_dir, None).unwrap(); - eprintln!("Generated artifacts written to: {}", base.display()); } diff --git a/tests/smoke.rs b/tests/smoke.rs index cab9d0f..4562a82 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -160,18 +160,6 @@ fn smoke_deployment_contract() { ); } -/// Helm chart generates from contract without panicking. -#[test] -fn smoke_helm_generation() { - let contract = Config::deployment_contract(); - let tmp = tempfile::tempdir().expect("tempdir"); - let result = scalo::deployment::generate_chart(&contract, tmp.path(), None); - assert!( - result.is_ok(), - "chart generation should succeed: {result:?}" - ); -} - /// Dockerfile generates from contract without panicking. #[test] fn smoke_dockerfile_generation() {