diff --git a/Cargo.lock b/Cargo.lock index e834b9dc..9958f1bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2944,6 +2944,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", + "percent-encoding", "pglz", "postgres-protocol", "regex-automata", @@ -2959,6 +2960,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "url", "wal-rus", "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index 3d461271..ded87fd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ futures = "0.3" libc = "0.2" lz4 = "1" pglz = "0.1" +percent-encoding = "2" postgres-protocol = "0.6" globset = "0.4" regex-automata = "0.4" @@ -68,6 +69,7 @@ tokio-util = { version = "0.7", features = ["io"] } toml = { version = "1", default-features = false, features = ["parse", "display", "serde"] } tracing = "0.1" tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "ansi"] } +url = "2" zstd = "0.13" opentelemetry = "0.32" opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] } diff --git a/README.md b/README.md index 80353f65..e02f8151 100644 --- a/README.md +++ b/README.md @@ -12,56 +12,44 @@ docs indexed at [plans/INDEX.md](plans/INDEX.md); for diagrams, ## Quick start (docker) -One command against your source PG (image publishing soon): +Point walshadow at source PostgreSQL and destination ClickHouse: ``` -docker run --rm \ - -v walshadow-data:/var/lib/walshadow \ - -v /etc/walshadow/ch.toml:/etc/walshadow/ch.toml:ro \ - clickhouse/walshadow \ - --host source.example --user replicator \ - --out-dir /var/lib/walshadow/wal \ - --spill-dir /var/lib/walshadow/spill \ - --shadow-socket-dir /var/run/postgresql \ - --bootstrap-shadow-data-dir /var/lib/walshadow/shadow \ - --walsender-bind 127.0.0.1:6510 \ - --ch-config /etc/walshadow/ch.toml -``` - -`ch.toml` is just the CH connection block (see [CH emitter -config](#ch-emitter-config)). walshadow bootstraps its own shadow PG, copies -every table into ClickHouse, then streams live — no per-table config. See -[Running standalone](#running-standalone) for the flags. - -## Try the demo locally +git submodule update --init --recursive -Full source PG → walshadow → ClickHouse stack (builds from source): +export PG_MAJOR=17 +export WALSHADOW_SOURCE_URL='postgres://replicator:secret@db.internal:5432/app?sslmode=require' +export WALSHADOW_CH_URL='clickhouse://default:secret@ch.internal:9000/cdc' -``` -git submodule update --init --recursive -docker compose -f docker/docker-compose.yml up --build -d +docker compose -f docker/docker-compose.yml build +docker compose -f docker/docker-compose.yml run --rm walshadow \ + init --all-tables +docker compose -f docker/docker-compose.yml up -d docker compose -f docker/docker-compose.yml logs -f walshadow ``` -Wait for the `shadow caught up to bootstrap end_lsn` line, then drive a change -and read it back: +Set `PG_MAJOR` to source PostgreSQL major. `init` validates both connections, +creates destination database, and selects source tables with row keys. Fix any +reported source requirements, then rerun it + +Wait for `shadow caught up to bootstrap end_lsn`, change a selected source row, +then query matching ClickHouse table: ``` -docker compose -f docker/docker-compose.yml exec source \ - psql -U postgres -c "UPDATE demo.users SET email='new@addr' WHERE id=1" -docker compose -f docker/docker-compose.yml exec clickhouse \ - clickhouse-client --query "SELECT id, email FROM demo.users FINAL ORDER BY id" +clickhouse-client --host ch.internal --database cdc --query \ + "SELECT * FROM users FINAL ORDER BY id" ``` -Full sequence in [docker/DEMO.md](docker/DEMO.md). For pgbench load with -Grafana dashboards and live schema-change propagation: +See [docker/QUICKSTART.md](docker/QUICKSTART.md) for table selection and +teardown. Add browser status with provisioned Prometheus and Grafana: ``` -docker compose -f docker/docker-compose.yml -f docker/docker-compose.demo.yml up --build -d +docker compose -f docker/docker-compose.yml \ + -f docker/docker-compose.grafana.yml up -d ``` -then open http://localhost:3000. Walkthrough in -[docker/DEMO.md](docker/DEMO.md) +Open http://localhost:3000 to inspect health, lag, throughput, queues, memory, +backfills, and source-transition state ## Source PG requirements @@ -94,7 +82,8 @@ create the target CH database (walshadow makes tables, not databases), then: ``` walshadow-stream \ - --host source.example --user replicator \ + --source-url postgres://replicator@source.example/app \ + --ch-url clickhouse://default@ch.example:9000/cdc \ --out-dir /var/lib/walshadow/wal \ --spill-dir /var/lib/walshadow/spill \ --shadow-socket-dir /var/run/postgresql \ @@ -103,7 +92,10 @@ walshadow-stream \ --ch-config /etc/walshadow/ch.toml ``` -with a `ch.toml` that is just the connection block: +Both URLs also read from `WALSHADOW_SOURCE_URL` / `WALSHADOW_CH_URL`, and both +decompose into the discrete `--host` / `--port` / … flags. `walshadow-stream +init` writes the config file; see [docker/QUICKSTART.md](docker/QUICKSTART.md). +`ch.toml` is the connection block: ```toml [ch] @@ -121,12 +113,33 @@ Notes: (it is baked into shadow's `primary_conninfo`). - `--bootstrap-shadow-data-dir` must be a new/empty dir; an initialized one resumes instead of re-bootstrapping. -- Without `--ch-config` the daemon stays metrics-only (no CH emission). Pass - `--metrics-bind 127.0.0.1:9484` for a Prometheus scrape endpoint. +- With neither `--ch-url` nor `[ch]` in `--ch-config`, the daemon stays + metrics-only (no CH emission). Pass `--metrics-bind 127.0.0.1:9484` for a + Prometheus scrape endpoint. +- `--ch-config` names a file that need not exist yet: the control socket + writes its fragments into the sibling `ch-config.d/`. - To manage shadow PG yourself, drop `--bootstrap-shadow-data-dir` (bootstrap defaults to `off` then, streaming only). See `walshadow-stream --help` for the full surface (bootstrap modes, walsender tuning, retention, etc.) +### Live control + +`--control-socket` opens a management socket the same binary speaks: + +``` +walshadow-stream ctl status # lag, rows synced, pause state +walshadow-stream ctl tables # source tables, `*` = replicated +walshadow-stream ctl add public users # opt in, CH table auto-creates +walshadow-stream ctl pause # freeze WAL consumption +walshadow-stream ctl source postgres://… # repoint the source endpoint +walshadow-stream ctl help +``` + +Each verb applies to the running session, which reconfigures in place with +no restart. Mutations land in `ch-config.d/50-api.toml`, leaving +operator-owned config untouched. Details in +[plans/control.md](plans/control.md) + ### CH emitter config `[ch]` connection defaults: `port = 9000`, `user = "default"`, empty @@ -146,6 +159,13 @@ replicate = false # Or list explicitly — replicate only what you name: [stream] replicate_all = false + +# Opt-in: shape comes from the source descriptor, CH table auto-creates +[table.public.orders] +replicate = true +initial_load = "copy" + +# Pinned: exact columns, nothing outside this list replicates [table.public.users] replicate = true initial_load = "none" @@ -163,8 +183,9 @@ carrying a dot or other TOML-special character quotes per key rules, e.g. `replicate_all` skips system schemas (`pg_*`, `information_schema`, the `[runtime_config]` schema). `attnum` values match `pg_attribute.attnum` (1-based) on the source relation; `type` is the CH destination type walshadow -advertises in the INSERT block. SIGHUP reloads mappings atomically; -connection params stay boot-only. +advertises in the INSERT block. SIGHUP (or `ctl reload`) re-reads the file: +mappings swap atomically, and a changed source or destination endpoint is +redialled without a restart Name a set of tables instead of one, with `match`: @@ -271,7 +292,7 @@ pgext/ walshadow decode-bridge PG module (PGXS) sql/ runtime-config overlay install SQL architecture/ overview + internals diagrams plans/ component design docs (overview.md is the baseline) -docker/ docker-compose demo + Dockerfile +docker/ quickstart Compose file + Dockerfile bench/ throughput / latency benchmark harnesses tests/ integration suite fixtures/wal/ golden WAL fixtures for offline tests diff --git a/bench/ec2/ec2-source-pg/cloud-init.yaml b/bench/ec2/ec2-source-pg/cloud-init.yaml index 6b7816c4..ada44431 100644 --- a/bench/ec2/ec2-source-pg/cloud-init.yaml +++ b/bench/ec2/ec2-source-pg/cloud-init.yaml @@ -1,8 +1,7 @@ #cloud-config # Provisions the walshadow "source" Postgres on a fresh Ubuntu host: # installs Docker, then runs postgres:17-bookworm with wal_level=logical -# and the init/source scripts (demo.users seed + replication HBA), -# mirroring the `source` service in docker/docker-compose.yml. +# with demo.users seed and replication HBA write_files: - path: /opt/walshadow/init/source/00-hba.sh permissions: '0755' diff --git a/bench/src/bin/local_bench.rs b/bench/src/bin/local_bench.rs index da302b11..1f4ccc8c 100644 --- a/bench/src/bin/local_bench.rs +++ b/bench/src/bin/local_bench.rs @@ -1,6 +1,5 @@ -//! Replication-latency benchmarks against the local docker-compose stack -//! (`docker/docker-compose.yml`, over host-exposed ports). The benchmark -//! engine and the full CLI surface live in `../bench.rs`, shared with +//! Replication-latency benchmarks against services on host-exposed ports +//! Benchmark engine and full CLI surface live in `../bench.rs`, shared with //! `ec2_bench`; the only thing this binary does differently is default the //! endpoints to localhost. //! @@ -18,7 +17,7 @@ use walshadow_bench::CommonArgs; #[derive(Parser, Debug)] #[command( name = "walshadow-local-bench", - about = "Measure source-Postgres → ClickHouse replication latency (local compose stack)", + about = "Measure source-Postgres → ClickHouse replication latency on local endpoints", // CommonArgs leaves --bench optional for ec2_bench's whole-suite mode; this // binary has no suite, so demand it at parse time. group(ArgGroup::new("what").args(["bench"]).required(true)), @@ -31,7 +30,7 @@ struct Args { #[tokio::main(flavor = "multi_thread")] async fn main() -> Result<()> { let args = Args::parse(); - // Local stack: source + destination default to localhost unless overridden + // Source and destination default to localhost unless overridden // (--ch-host doubles as the destination-host override here). let pg_host = args .common diff --git a/docker/DEMO.md b/docker/DEMO.md deleted file mode 100644 index 9d7b4804..00000000 --- a/docker/DEMO.md +++ /dev/null @@ -1,196 +0,0 @@ -# DEMO — walshadow live, in a browser - -Source PG → walshadow → ClickHouse, with **pgbench hammering the source** -and **Grafana dashboards** showing throughput, replication lag, and rows -landing in ClickHouse in near-real-time — then an operator evolving the -schema live and watching the column appear downstream. - -Five services from the base stack plus a demo tier: - -| service | role | -|---|---| -| `source` | postgres:18, `wal_level=logical`, seeds `demo.users` + pgbench TPC-B schema (`REPLICA IDENTITY FULL`) | -| `walshadow` | daemon: in-container daemon-owned shadow PG + WAL→CH stream, `/metrics` on :9484 | -| `clickhouse` | destination; only the `demo` database is pre-created, walshadow auto-creates the tables | -| `pgbench` | hammers `source` with the TPC-B workload | -| `postgres-exporter` | source PG stats (TPS, tuple rates) → Prometheus | -| `prometheus` | scrapes walshadow + postgres-exporter | -| `grafana` | the dashboards — http://localhost:3000 | - -> The demo tier lives in an overlay file. Everything below uses both -> compose files. Set a shell alias once and reuse it: -> -> ``` -> dc="docker compose -f docker/docker-compose.yml -f docker/docker-compose.demo.yml" -> ``` - -## 1. Bring up the stack - -``` -git submodule update --init --recursive -$dc up --build -d -``` - -First build is heavy (Rust release + PGXS shared object); subsequent -`up`s reuse layers. Grafana pulls the `grafana-clickhouse-datasource` -plugin on first boot, so the first `up` needs internet. - -> The pgbench schema is seeded by one-shot init scripts that run only -> when the data volumes are empty. If you previously ran the lean base -> stack, drop its volumes first so the demo tables get created: -> `$dc down -v` before the `up` above. - -Tunable load (defaults shown), set before `up`: - -``` -PGBENCH_SCALE=1 PGBENCH_CLIENTS=4 PGBENCH_THREADS=2 $dc up --build -d -``` - -`PGBENCH_SCALE=1` is ~100k accounts; bump it for a bigger backfill and a -heavier hammer. - -## 2. Watch bootstrap land - -``` -$dc logs -f walshadow -``` - -Wait for the four bootstrap phase lines, ending with: - -``` -walshadow::bootstrap: shadow caught up to bootstrap end_lsn -``` - -The `pgbench` service is gated on walshadow's metrics port, which opens -only *after* bootstrap — so the hammer starts swinging the moment the -backfill is durable. Confirm it's swinging: - -``` -$dc logs -f pgbench # progress lines every 5s: tps, latency -``` - -## 3. Open the dashboards - -Browse to **http://localhost:3000** (anonymous admin — no login). It -lands on **walshadow — live CDC pipeline**. Set the top-right refresh to -`2s` / `5s` and the range to `Last 5 minutes`. - -Five sections, top to bottom: - -1. **pgbench → PostgreSQL · the hammer** — source commit TPS and tuple - write rate (insert/update/delete) from `postgres-exporter`. This is - the load going in. -2. **walshadow pipeline** — heap records decoded/s, transactions - committed/s, and filtered WAL records/s broken out by resource - manager (Heap / Transaction / Btree / …). Throughput *through* - walshadow. -3. **replication lag · latency** — shadow apply lag in seconds (the - headline latency number) and the byte backlog between source, shadow - PG, and the ClickHouse ack point. Under steady load this hugs zero - and snaps back after any burst. -4. **buffers & memory** — xact-buffer bytes in memory vs spilled to - disk, active buffered xacts, aborts, spill evictions. -5. **ClickHouse destination · rows landed** — queried straight from CH: - cumulative `pgbench_history` rows, the per-second insert rate landing - in ClickHouse (compare its shape to section 1's source TPS), and a - live `demo.users` table — the one that grows a column in step 4. - -Direct link to the dashboard: http://localhost:3000/d/walshadow-live - -Let it run a minute. The CH "rows landed /s" bars should track the -source TPS line with the lag shown in section 3. - -## 4. Drive a row change, then evolve the schema (live) - -`demo.users` is tiny, has just `id` / `name` / `email`, and is untouched -by pgbench — so it's the clean stage for showing CDC and live DDL -replication while the hammer roars in the background. walshadow -auto-created it and copied its rows at bootstrap (`replicate_all`). - -**Beat 1 — a row change rides the stream.** Update a row on the source -and read it back from ClickHouse: - -``` -$dc exec source psql -U postgres -d postgres -c \ - "UPDATE demo.users SET email='opifex@merces-digna' WHERE id=1" - -$dc exec clickhouse clickhouse-client --query \ - "SELECT id, email, _lsn FROM demo.users FINAL ORDER BY id" -``` - -Row 1's email updates; its `_lsn` advances — CDC in flight. - -**Beat 2 — the schema evolves.** Add a column on the source and watch -walshadow replicate the DDL to ClickHouse — no config edit, no restart: - -``` -$dc exec source psql -U postgres -d postgres \ - -c "ALTER TABLE demo.users ADD COLUMN signup_ts timestamptz" \ - -c "UPDATE demo.users SET signup_ts = now()" - -$dc exec clickhouse clickhouse-client --query "DESCRIBE TABLE demo.users" -``` - -`signup_ts` appears as `Nullable(DateTime64(6, 'UTC'))` — walshadow ran -the `ALTER TABLE … ADD COLUMN` on ClickHouse the instant it decoded the -source DDL, then auto-extended the column mapping so the `UPDATE`'s -values ship too: - -``` -$dc exec clickhouse clickhouse-client --query \ - "SELECT id, name, signup_ts FROM demo.users FINAL ORDER BY id" -``` - -The Grafana **demo.users (live schema)** panel (section 5) shows the -same thing without leaving the browser: the new `signup_ts` column pops -into the table on the next refresh. - -Want to evolve a *hot* table too? It's already streaming, so its shape -is known — `ALTER TABLE pgbench_accounts ADD COLUMN region text DEFAULT -'eu'` on the source propagates the same way mid-hammer; watch -`DESCRIBE TABLE demo.pgbench_accounts` on CH grow the column. - -## 5. Teardown - -``` -$dc down -v --remove-orphans -``` - -`-v` drops the named volumes (`source-data`, `clickhouse-data`, -`walshadow-data`); next `up` rebootstraps from scratch. - ---- - -## Appendix — CLI verification (no browser) - -The pipeline is fully inspectable from the shell. - -Snapshot of a streamed pgbench table: - -``` -$dc exec clickhouse clickhouse-client --query \ - "SELECT count() FROM demo.pgbench_history" -$dc exec clickhouse clickhouse-client --query \ - "SELECT aid, abalance, _lsn FROM demo.pgbench_accounts FINAL ORDER BY aid LIMIT 5" -``` - -Shadow PG (in-container standby) replay position: - -``` -$dc exec walshadow psql -h /var/run/postgresql -U postgres \ - -c "SELECT pg_is_in_recovery(), pg_last_wal_replay_lsn()" -``` - -Source-side LSN + replication state (note: slotless physical -replication — `pg_replication_slots` stays empty): - -``` -$dc exec source psql -U postgres -c \ - "SELECT pid, state, sent_lsn, write_lsn, replay_lsn FROM pg_stat_replication" -``` - -Raw metrics the dashboards are built on: - -``` -curl -s http://localhost:9484/metrics | grep -E '^walshadow_(decoder_decoded|xacts_committed|shadow_apply_lag)' -``` diff --git a/docker/Dockerfile b/docker/Dockerfile index d667b435..7014ab34 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,5 @@ -# walshadow demo image. Multi-stage on alpine/musl: +# walshadow image +# Multi-stage on alpine/musl: # 1. rust:1-alpine -> static-ish walshadow-stream binary. alpine 3.23 # ships gcc 15, so clickhouse-c-rs builds under -std=c23. # 2. postgres:N-alpine -> walshadow PG module, compiled against the @@ -6,6 +7,10 @@ # binary itself links no PG, so it builds separately. # 3. postgres:N-alpine runtime carries both, so daemon-owned shadow PG # resolves `shared_preload_libraries = 'walshadow'` from its own $libdir. +# +# PG_MAJOR must equal the source cluster's major: the shadow is a physical +# clone, and a basebackup cannot span majors. Build for another source with +# `--build-arg PG_MAJOR=17`. ARG PG_MAJOR=18 diff --git a/docker/QUICKSTART.md b/docker/QUICKSTART.md new file mode 100644 index 00000000..38db9889 --- /dev/null +++ b/docker/QUICKSTART.md @@ -0,0 +1,89 @@ +# Connect existing databases + +Requires Docker Compose, PostgreSQL 16 or newer, and ClickHouse. Set +`PG_MAJOR` to source PostgreSQL major because shadow is its physical clone + +``` +git submodule update --init --recursive + +export PG_MAJOR=17 +export WALSHADOW_SOURCE_URL='postgres://replicator:secret@db.internal:5432/app?sslmode=require' +export WALSHADOW_CH_URL='clickhouse://default:secret@ch.internal:9000/cdc' + +docker compose -f docker/docker-compose.yml build +docker compose -f docker/docker-compose.yml run --rm walshadow \ + init --all-tables +docker compose -f docker/docker-compose.yml up -d +docker compose -f docker/docker-compose.yml logs -f walshadow +``` + +`init` validates both connections, creates destination database, and selects +source tables with row keys. It reports SQL needed when source does not have +`wal_level = logical`, replication permission, or usable row keys + +Wait for: + +``` +walshadow::bootstrap: shadow caught up to bootstrap end_lsn +``` + +Stop following logs, change a selected source table, then query matching +ClickHouse table: + +``` +clickhouse-client --host ch.internal --database cdc --query \ + "SELECT * FROM users FINAL ORDER BY id" +``` + +Destination table includes `_lsn`. `FINAL` returns latest row version + +## Browser status + +Start optional Prometheus and Grafana layer: + +``` +docker compose -f docker/docker-compose.yml \ + -f docker/docker-compose.grafana.yml up -d +``` + +Open http://localhost:3000. Provisioned dashboard shows daemon health, +replication lag, ClickHouse acknowledgement backlog, throughput, queue depth, +memory and spill use, backfills, and timeline or endpoint-swap problems. Set +`WALSHADOW_GRAFANA_PORT` before `up` to use another host port + +Grafana reads only walshadow's Prometheus endpoint. It does not connect to +source PostgreSQL or destination ClickHouse + +## Select tables + +``` +docker compose -f docker/docker-compose.yml run --rm walshadow init +``` + +Interactive mode lists source tables. Select by number or enter `all`. +Non-interactive selection accepts repeated table names: + +``` +docker compose -f docker/docker-compose.yml run --rm walshadow \ + init --table public users --table public orders +``` + +`--initial-load copy` is default. Use `--initial-load none` to stream only +changes after selection. Config persists in `walshadow-config` volume + +## Teardown + +``` +docker compose -f docker/docker-compose.yml down -v +``` + +`-v` removes local shadow and config volumes. It does not modify source or +remove ClickHouse tables + +When browser status layer is running, include its Compose file so Prometheus +and Grafana are removed too: + +``` +docker compose -f docker/docker-compose.yml \ + -f docker/docker-compose.grafana.yml down -v +``` diff --git a/docker/ch-config.demo.toml b/docker/ch-config.demo.toml deleted file mode 100644 index 20a95fc1..00000000 --- a/docker/ch-config.demo.toml +++ /dev/null @@ -1,49 +0,0 @@ -# walshadow demo CH emitter mapping — observability variant. replicate_all -# (on by default) auto-creates demo.users; the explicit pgbench mappings pin -# dest types to the tables pre-created by init/clickhouse/02-pgbench.sh. - -[ch] -host = "clickhouse" -port = 9000 -database = "demo" -user = "default" -password = "" -compression = "lz4" -flush_timeout_ms = 200 - -[table.public.pgbench_accounts] -target_database = "demo" -columns = [ - { attnum = 1, target = "aid", type = "Int32" }, - { attnum = 2, target = "bid", type = "Int32" }, - { attnum = 3, target = "abalance", type = "Int32" }, - { attnum = 4, target = "filler", type = "String" }, -] - -[table.public.pgbench_branches] -target_database = "demo" -columns = [ - { attnum = 1, target = "bid", type = "Int32" }, - { attnum = 2, target = "bbalance", type = "Int32" }, - { attnum = 3, target = "filler", type = "Nullable(String)" }, -] - -[table.public.pgbench_tellers] -target_database = "demo" -columns = [ - { attnum = 1, target = "tid", type = "Int32" }, - { attnum = 2, target = "bid", type = "Int32" }, - { attnum = 3, target = "tbalance", type = "Int32" }, - { attnum = 4, target = "filler", type = "Nullable(String)" }, -] - -[table.public.pgbench_history] -target_database = "demo" -columns = [ - { attnum = 1, target = "tid", type = "Int32" }, - { attnum = 2, target = "bid", type = "Int32" }, - { attnum = 3, target = "aid", type = "Int32" }, - { attnum = 4, target = "delta", type = "Int32" }, - { attnum = 5, target = "mtime", type = "DateTime64(6)" }, - { attnum = 6, target = "filler", type = "Nullable(String)" }, -] diff --git a/docker/ch-config.toml b/docker/ch-config.toml deleted file mode 100644 index 71669dc3..00000000 --- a/docker/ch-config.toml +++ /dev/null @@ -1,20 +0,0 @@ -# walshadow CH emitter config. With replicate_all on (the default) a bare [ch] -# block replicates every user table into [ch] database, which must already exist -# (walshadow creates tables, not databases). - -[ch] -host = "clickhouse" -port = 9000 -database = "demo" -user = "default" -password = "" -compression = "lz4" -# soft_delete = true - -# Opt a table out: -# [table.public.audit_log] -# replicate = false - -# Explicit mode — replicate only what you list: -# [stream] -# replicate_all = false diff --git a/docker/docker-compose.demo.yml b/docker/docker-compose.demo.yml deleted file mode 100644 index 3e319581..00000000 --- a/docker/docker-compose.demo.yml +++ /dev/null @@ -1,105 +0,0 @@ -# walshadow dashboard demo overlay. Layer ON TOP of docker-compose.yml: -# -# docker compose -f docker/docker-compose.yml \ -# -f docker/docker-compose.demo.yml up --build -d -# -# Adds the observability + load-generation tier on top of the lean -# source→walshadow→clickhouse base: -# pgbench — hammers source with the TPC-B workload -# postgres-exporter — source PG stats (TPS, tuple write rates) → Prom -# prometheus — scrapes walshadow:9484 + postgres-exporter:9187 -# grafana — http://localhost:3000, anonymous-admin, lands on -# the "walshadow — live CDC pipeline" dashboard -# -# The base services are extended (not replaced): the env toggles below -# arm the demo-only init steps (init/source/02-pgbench.sh, -# init/clickhouse/02-pgbench.sh — no-ops without WALSHADOW_DEMO_PGBENCH) -# and repoint walshadow at the pgbench-aware ch-config + a 1s status -# cadence so the graphs move in near-real-time. - -services: - source: - environment: - WALSHADOW_DEMO_PGBENCH: "1" - PGBENCH_SCALE: "${PGBENCH_SCALE:-1}" - - clickhouse: - environment: - WALSHADOW_DEMO_PGBENCH: "1" - - walshadow: - environment: - RUST_LOG: warn,walshadow=info - WALSHADOW_STATUS_INTERVAL: "1" - WALSHADOW_CH_CONFIG: /etc/walshadow/ch-config.demo.toml - volumes: - - ./ch-config.demo.toml:/etc/walshadow/ch-config.demo.toml:ro - - pgbench: - image: postgres:18-bookworm - depends_on: - source: - condition: service_healthy - walshadow: - condition: service_started - environment: - PGBENCH_CLIENTS: "${PGBENCH_CLIENTS:-4}" - PGBENCH_THREADS: "${PGBENCH_THREADS:-2}" - PGBENCH_DURATION: "${PGBENCH_DURATION:-86400}" - entrypoint: ["/bin/bash", "-c"] - # `$$` escapes the compose-interpolation pass so these expand from - # the container's env at runtime, not from the host shell at config - # time (which would blank them). - command: - - | - set -euo pipefail - echo "pgbench: waiting for source PG ..." - until pg_isready -h source -U postgres -d postgres >/dev/null 2>&1; do sleep 1; done - echo "pgbench: waiting for walshadow bootstrap (metrics port 9484) ..." - until (echo > /dev/tcp/walshadow/9484) 2>/dev/null; do sleep 1; done - sleep 2 - echo "pgbench: hammering source (-c $${PGBENCH_CLIENTS} -j $${PGBENCH_THREADS} for $${PGBENCH_DURATION}s)" - exec pgbench -h source -U postgres -d postgres -n \ - -c "$${PGBENCH_CLIENTS}" -j "$${PGBENCH_THREADS}" \ - -T "$${PGBENCH_DURATION}" -P 5 - - postgres-exporter: - image: quay.io/prometheuscommunity/postgres-exporter:v0.19.1 - depends_on: - source: - condition: service_healthy - environment: - DATA_SOURCE_NAME: "postgresql://postgres@source:5432/postgres?sslmode=disable" - ports: - - "9187:9187" - - prometheus: - image: prom/prometheus:v3.12.0 - depends_on: - - postgres-exporter - volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.retention.time=1h - ports: - - "9090:9090" - - grafana: - image: grafana/grafana:13.0.2 - depends_on: - - prometheus - - clickhouse - environment: - GF_INSTALL_PLUGINS: grafana-clickhouse-datasource - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: Admin - GF_AUTH_DISABLE_LOGIN_FORM: "true" - GF_USERS_DEFAULT_THEME: dark - GF_DASHBOARDS_MIN_REFRESH_INTERVAL: "1s" - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/walshadow.json - volumes: - - ./grafana/provisioning:/etc/grafana/provisioning:ro - - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - ports: - - "3000:3000" diff --git a/docker/docker-compose.grafana.yml b/docker/docker-compose.grafana.yml new file mode 100644 index 00000000..cbdf97b5 --- /dev/null +++ b/docker/docker-compose.grafana.yml @@ -0,0 +1,32 @@ +# Optional browser status layer for docker/docker-compose.yml + +services: + prometheus: + image: prom/prometheus:v3.12.0 + depends_on: + walshadow: + condition: service_started + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=24h + volumes: + - ./grafana/prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "${WALSHADOW_PROMETHEUS_PORT:-9090}:9090" + + grafana: + image: grafana/grafana:13.0.2 + depends_on: + - prometheus + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/walshadow.json + GF_DASHBOARDS_MIN_REFRESH_INTERVAL: "1s" + GF_USERS_DEFAULT_THEME: dark + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: + - "${WALSHADOW_GRAFANA_PORT:-3000}:3000" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ac477d00..96710675 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,112 +1,44 @@ -# walshadow demo. Three services: -# source — postgres:18 with wal_level=logical, demo.users seeded -# via init script, REPLICA IDENTITY FULL set -# clickhouse — clickhouse-server; only the demo database is pre-created, -# walshadow auto-creates the tables (replicate_all, on by default) -# walshadow — custom image carrying walshadow-stream + the walshadow PG -# extension. Boots an in-container shadow PG on the empty volume, -# copies every source table into clickhouse, then streams live; -# resumes from the cursor on later starts +# walshadow quickstart — replicate your own Postgres into your own +# ClickHouse. One service: the daemon, plus the shadow PG it owns inside the +# container. Walkthrough in docker/QUICKSTART.md # -# Quick start (from repo root): -# git submodule update --init --recursive -# docker compose -f docker/docker-compose.yml up --build +# export WALSHADOW_SOURCE_URL=postgres://user:password@host:5432/dbname +# export WALSHADOW_CH_URL=clickhouse://user:password@host:9000/database +# docker compose -f docker/docker-compose.yml run --rm walshadow init +# docker compose -f docker/docker-compose.yml up --build -d # docker compose -f docker/docker-compose.yml logs -f walshadow # wait for bootstrap +# docker compose -f docker/docker-compose.yml \ +# -f docker/docker-compose.grafana.yml up -d # browser status # -# Drive the pipeline: -# docker compose -f docker/docker-compose.yml exec source \ -# psql -U postgres -c "UPDATE demo.users SET email='opifex@merces-digna' WHERE id=1" -# docker compose -f docker/docker-compose.yml exec clickhouse \ -# clickhouse-client --query "SELECT * FROM demo.users FINAL" - -services: - source: - image: postgres:18-bookworm - environment: - POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: trust - command: - - postgres - - -c - - wal_level=logical - - -c - - max_wal_senders=8 - - -c - - max_replication_slots=8 - - -c - - wal_keep_size=128MB - - -c - - wal_compression=lz4 - volumes: - - ./init/source:/docker-entrypoint-initdb.d:ro - # PG 18+ official image stores data in a major-version subdir - # (/var/lib/postgresql/18/docker); mount the parent, not /data. - # See https://github.com/docker-library/postgres/pull/1259 - - source-data:/var/lib/postgresql - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] - interval: 2s - timeout: 3s - retries: 30 - ports: - - "5432:5432" - - clickhouse: - image: clickhouse/clickhouse-server:26.3 - environment: - CLICKHOUSE_SKIP_USER_SETUP: "1" - volumes: - - ./init/clickhouse:/docker-entrypoint-initdb.d:ro - - clickhouse-data:/var/lib/clickhouse - ulimits: - nofile: - soft: 262144 - hard: 262144 - healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:8123/ping"] - interval: 2s - timeout: 3s - retries: 30 - ports: - - "8123:8123" - - "9000:9000" +# The shadow is a physical clone of the source, so the image's PostgreSQL +# major must equal the source's. Anything but 18: +# +# PG_MAJOR=17 docker compose -f docker/docker-compose.yml up --build -d - # End-to-end trace sink. Jaeger all-in-one ingests OTLP/gRPC on 4317 - # (COLLECTOR_OTLP_ENABLED) and serves the trace UI on 16686. Open a - # slow `txn` there to see its commit.drain → emit.insert waterfall. - jaeger: - image: jaegertracing/all-in-one:1.57 - environment: - COLLECTOR_OTLP_ENABLED: "true" - ports: - - "16686:16686" - - "4317:4317" +name: walshadow +services: walshadow: build: context: .. dockerfile: docker/Dockerfile - depends_on: - source: - condition: service_healthy - clickhouse: - condition: service_healthy - jaeger: - condition: service_started + args: + PG_MAJOR: ${PG_MAJOR:-18} environment: - RUST_LOG: warn,walshadow=info - WALSHADOW_SOURCE_HOST: source - WALSHADOW_SOURCE_PORT: "5432" - # Ship spans to the Jaeger sidecar. Unset this (or remove jaeger) - # to run with zero tracing overhead — no exporter is installed. - OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4317 + - RUST_LOG=${RUST_LOG:-warn,walshadow=info} + # Bare names pass the shell's value through, so an unset URL stays unset + - WALSHADOW_SOURCE_URL + - WALSHADOW_CH_URL + ports: + - "${WALSHADOW_METRICS_PORT:-9484}:9484" volumes: - - ./ch-config.toml:/etc/walshadow/ch-config.toml:ro + # WAL segments, spill files, and the shadow data dir. Losing it costs a + # rebootstrap, not correctness - walshadow-data:/var/lib/walshadow - ports: - - "9484:9484" + # `init` writes ch-config.toml here; `ctl` writes its fragments beside it + - walshadow-config:/etc/walshadow + stop_grace_period: 30s volumes: - source-data: - clickhouse-data: walshadow-data: + walshadow-config: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7471810d..40d8bf46 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,9 +1,23 @@ #!/usr/bin/env bash -# walshadow demo entrypoint. Creates state directories, then execs daemon -# against docker-compose source and ClickHouse services +# walshadow container entrypoint. Creates state directories, then execs the +# daemon against the configured source and ClickHouse. +# +# Connection settings come from WALSHADOW_SOURCE_URL / WALSHADOW_CH_URL (the +# daemon reads both from the environment), from a config mounted at +# WALSHADOW_CH_CONFIG, or from the discrete WALSHADOW_SOURCE_* variables. +# +# `init` and `ctl` run as themselves, so `docker compose run --rm walshadow +# init` and `docker compose exec walshadow walshadow-stream ctl status` both +# work without the daemon's flags. set -euo pipefail +case "${1:-}" in + init | ctl) + exec walshadow-stream "$@" + ;; +esac + SHADOW_DATA="${WALSHADOW_SHADOW_DATA:-/var/lib/walshadow/shadow-data}" OUT_DIR="${WALSHADOW_OUT_DIR:-/var/lib/walshadow/out}" SPILL_DIR="${WALSHADOW_SPILL_DIR:-/var/lib/walshadow/spill}" @@ -19,6 +33,28 @@ mkdir -p "$OUT_DIR" "$SPILL_DIR" "$SOCKET_DIR" CH_CONFIG="${WALSHADOW_CH_CONFIG:-/etc/walshadow/ch-config.toml}" mkdir -p "${CH_CONFIG%.toml}.d" +# Discrete source flags for callers predating WALSHADOW_SOURCE_URL +# (bench/ec2/ec2-walshadow/deploy.sh). Skipped once a URL is set, which the +# daemon reads for itself. +SOURCE_ARGS=() +if [ -z "${WALSHADOW_SOURCE_URL:-}" ]; then + if [ -z "${WALSHADOW_SOURCE_HOST:-}" ] && [ ! -f "$CH_CONFIG" ]; then + echo "walshadow: no source configured. Set WALSHADOW_SOURCE_URL," >&2 + echo " eg postgres://user:password@host:5432/dbname, or mount a" >&2 + echo " config at $CH_CONFIG (write one with \`init\`)." >&2 + exit 64 + fi + if [ -n "${WALSHADOW_SOURCE_HOST:-}" ]; then + SOURCE_ARGS+=( + --host "$WALSHADOW_SOURCE_HOST" + --port "${WALSHADOW_SOURCE_PORT:-5432}" + --user "${WALSHADOW_SOURCE_USER:-postgres}" + --dbname "${WALSHADOW_SOURCE_DB:-postgres}" + --sslmode "${WALSHADOW_SOURCE_SSLMODE:-disable}" + ) + fi +fi + # Pool sizes fall through to the binary's compiled defaults unless overridden # via env. clap rejects a flag passed twice, so only inject when the caller # (e.g. EC2 deploy.sh via "$@") didn't already pass it. @@ -41,11 +77,7 @@ case " $* " in esac exec walshadow-stream \ - --host "${WALSHADOW_SOURCE_HOST:-source}" \ - --port "${WALSHADOW_SOURCE_PORT:-5432}" \ - --user "${WALSHADOW_SOURCE_USER:-postgres}" \ - --dbname "${WALSHADOW_SOURCE_DB:-postgres}" \ - --sslmode disable \ + "${SOURCE_ARGS[@]}" \ --out-dir "$OUT_DIR" \ --spill-dir "$SPILL_DIR" \ --shadow-socket-dir "$SOCKET_DIR" \ @@ -55,7 +87,7 @@ exec walshadow-stream \ --bootstrap-mode direct \ --bootstrap-shadow-data-dir "$SHADOW_DATA" \ --walsender-bind 127.0.0.1:5433 \ - --ch-config "${WALSHADOW_CH_CONFIG:-/etc/walshadow/ch-config.toml}" \ + --ch-config "$CH_CONFIG" \ --metrics-bind 0.0.0.0:9484 \ --control-socket "${WALSHADOW_CONTROL_SOCKET:-/var/run/walshadow/control.sock}" \ --status-interval "${WALSHADOW_STATUS_INTERVAL:-5}" \ diff --git a/docker/grafana/dashboards/walshadow.json b/docker/grafana/dashboards/walshadow.json index b85fe668..c91ddf63 100644 --- a/docker/grafana/dashboards/walshadow.json +++ b/docker/grafana/dashboards/walshadow.json @@ -1,1993 +1,1035 @@ { - "uid": "walshadow-live", - "title": "walshadow \u2014 live CDC pipeline", - "tags": [ - "walshadow", - "cdc", - "clickhouse" - ], - "timezone": "", - "editable": true, - "schemaVersion": 39, - "refresh": "2s", - "time": { - "from": "now-5m", - "to": "now" - }, - "annotations": { - "list": [] - }, - "templating": { - "list": [] - }, - "panels": [ - { - "id": 100, - "type": "row", - "title": "\u2460 PostgreSQL", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "panels": [] + "annotations": { + "list": [] }, - { - "id": 1, - "type": "stat", - "title": "Source TPS (commits/s)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "background", - "graphMode": "area", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(pg_stat_database_xact_commit{datname=\"postgres\"}[$__rate_interval])", - "legendFormat": "tps" - } - ] - }, - { - "id": 2, - "type": "stat", - "title": "Rows updated/s (source)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "background", - "graphMode": "area", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(pg_stat_database_tup_updated{datname=\"postgres\"}[$__rate_interval])", - "legendFormat": "updated/s" - } - ] - }, - { - "id": 3, - "type": "timeseries", - "title": "Source write rate (tuples/s)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 1 - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(pg_stat_database_tup_inserted{datname=\"postgres\"}[$__rate_interval])", - "legendFormat": "inserted" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(pg_stat_database_tup_updated{datname=\"postgres\"}[$__rate_interval])", - "legendFormat": "updated" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(pg_stat_database_tup_deleted{datname=\"postgres\"}[$__rate_interval])", - "legendFormat": "deleted" - } - ] - }, - { - "id": 101, - "type": "row", - "title": "\u2461 walshadow pipeline", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "panels": [] - }, - { - "id": 4, - "type": "timeseries", - "title": "Records decoded /s", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 7 - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 25, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_decoder_decoded_total[$__rate_interval])", - "legendFormat": "heap records/s" - } - ] - }, - { - "id": 5, - "type": "timeseries", - "title": "Transactions committed /s", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 7 - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 25, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ + "editable": false, + "graphTooltip": 1, + "links": [], + "panels": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_xacts_committed_total[$__rate_interval])", - "legendFormat": "xacts/s" - } - ] - }, - { - "id": 6, - "type": "timeseries", - "title": "Filter records /s by rmgr", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 7 - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 35, - "gradientMode": "opacity", - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_filter_records_total[$__rate_interval])", - "legendFormat": "{{rmgr}} \u00b7 {{route}}" - } - ] - }, - { - "id": 102, - "type": "row", - "title": "\u2462 replication lag", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "panels": [] - }, - { - "id": 7, - "type": "stat", - "title": "Apply lag (seconds)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 15 - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "decimals": 2, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "background", - "graphMode": "area", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_shadow_apply_lag_seconds", - "legendFormat": "lag" - } - ] - }, - { - "id": 8, - "type": "timeseries", - "title": "Shadow apply lag over time", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 6, - "y": 15 - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_shadow_apply_lag_seconds", - "legendFormat": "apply lag (s)" - } - ] - }, - { - "id": 9, - "type": "timeseries", - "title": "Pipeline backlog (bytes)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 15 - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_shadow_apply_lag_bytes", - "legendFormat": "source \u2192 shadow apply lag" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)", - "legendFormat": "source \u2192 CH-ack backlog" - } - ] - }, - { - "id": 103, - "type": "row", - "title": "\u2463 buffers & memory", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "panels": [] - }, - { - "id": 10, - "type": "timeseries", - "title": "Xact buffer bytes", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 22 - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_xact_bytes_in_memory", - "legendFormat": "in memory" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_spill_bytes_active", - "legendFormat": "spilled to disk" - } - ] - }, - { - "id": 11, - "type": "stat", - "title": "Active xacts buffered", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 22 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_xact_active", - "legendFormat": "active" - } - ] - }, - { - "id": 12, - "type": "stat", - "title": "Daemon uptime", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 22 - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_uptime_seconds", - "legendFormat": "uptime" - } - ] - }, - { - "id": 13, - "type": "stat", - "title": "Aborted xacts", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 22 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_xacts_aborted_total", - "legendFormat": "aborted" - } - ] - }, - { - "id": 14, - "type": "stat", - "title": "Spill evictions", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 22 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_spill_evictions_total", - "legendFormat": "evictions" - } - ] - }, - { - "id": 104, - "type": "row", - "title": "\u2464 ClickHouse", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "panels": [] - }, - { - "id": 15, - "type": "stat", - "title": "rows in demo.users (cumulative)", - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "walshadow-ch" - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 29 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "background", - "graphMode": "area", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "walshadow-ch" - }, - "editorType": "sql", - "queryType": "table", - "rawSql": "SELECT count() AS rows FROM demo.users" - } - ] - }, - { - "id": 16, - "type": "timeseries", - "title": "demo.users rows/s (by source commit time)", - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "walshadow-ch" - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 6, - "y": 29 - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 60, - "gradientMode": "hue", - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "walshadow-ch" - }, - "editorType": "sql", - "queryType": "timeseries", - "rawSql": "SELECT toStartOfInterval(_commit_ts, INTERVAL 2 SECOND) AS time, count() / 2 AS \"rows/s\" FROM demo.users WHERE $__timeFilter(_commit_ts) GROUP BY time ORDER BY time" - } - ] - }, - { - "id": 17, - "type": "table", - "title": "demo.users in ClickHouse (live schema)", - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "walshadow-ch" - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 29 - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "walshadow-ch" - }, - "editorType": "sql", - "queryType": "table", - "rawSql": "SELECT * FROM demo.users FINAL ORDER BY id LIMIT 100" - } - ] - }, - { - "id": 105, - "type": "row", - "title": "\u2465 inserter pipeline depth", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "panels": [] - }, - { - "id": 18, - "type": "timeseries", - "title": "Parser rows per second", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 36 - }, - "fieldConfig": { - "defaults": { - "unit": "rows", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "processed /s" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - } - ] - } - ] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "refId": "S", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "(rate(pg_stat_database_tup_inserted{datname=\"postgres\"}[$__rate_interval])+rate(pg_stat_database_tup_updated{datname=\"postgres\"}[$__rate_interval])+rate(pg_stat_database_tup_deleted{datname=\"postgres\"}[$__rate_interval]))", - "legendFormat": "source produced /s" - }, - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_decoder_decoded_total[$__rate_interval])", - "legendFormat": "parsed rows /s" - } - ] - }, - { - "id": 19, - "type": "timeseries", - "title": "ClickHouse rows backlog (ingester \u2212 landed)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 50 - }, - "fieldConfig": { - "defaults": { - "unit": "rows", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "clamp_min(walshadow_insertbatch_rows_in_total - walshadow_emitter_rows_total,0)", - "legendFormat": "rows queued for ClickHouse" - } - ] - }, - { - "id": 20, - "type": "timeseries", - "title": "ClickHouse rows /s (ingester \u2192 landed)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 43 - }, - "fieldConfig": { - "defaults": { - "unit": "rows", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_insertbatch_rows_in_total[$__rate_interval])", - "legendFormat": "rows \u2192 ingester /s" + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Status", + "type": "row" }, { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_emitter_rows_total[$__rate_interval])", - "legendFormat": "rows \u2192 ClickHouse /s" - } - ] - }, - { - "id": 24, - "type": "timeseries", - "title": "Queueing thread out", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 36 - }, - "fieldConfig": { - "defaults": { - "unit": "rows", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "Down" + }, + "1": { + "color": "green", + "text": "Running" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "up{job=\"walshadow\"}", + "instant": true, + "legendFormat": "walshadow", + "refId": "A" + } + ], + "title": "Metrics status", + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_decode_rows_out_total[$__rate_interval])", - "legendFormat": "rows out of decode pool /s" - } - ] - }, - { - "id": 25, - "type": "timeseries", - "title": "Decoding rows in flight", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 43 - }, - "fieldConfig": { - "defaults": { - "unit": "rows", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "walshadow_shadow_apply_lag_seconds", + "instant": true, + "legendFormat": "lag", + "refId": "A" + } + ], + "title": "Shadow apply lag", + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "clamp_min(walshadow_decoder_decoded_total - walshadow_decode_rows_out_total,0)", - "legendFormat": "rows in decode pool" - } - ] - }, - { - "id": 26, - "type": "timeseries", - "title": "ClickHouse input batcher", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 43 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 1073741824 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)", + "instant": true, + "legendFormat": "backlog", + "refId": "A" + } + ], + "title": "ClickHouse ack backlog", + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_decode_rows_out_total - walshadow_insertbatch_rows_in_total", - "legendFormat": "rows queued for batcher" - } - ] - }, - { - "id": 106, - "type": "row", - "title": "\u2466 row flow & pipeline tail", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 57 - }, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "Rows /s by stage (decoded \u2192 ingester \u2192 ClickHouse)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 58 - }, - "fieldConfig": { - "defaults": { - "unit": "rows", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "rate(walshadow_emitter_rows_total[$__rate_interval])", + "instant": true, + "legendFormat": "rows/s", + "refId": "A" + } + ], + "title": "ClickHouse rows/s", + "type": "stat" }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_decoder_decoded_total[$__rate_interval])", - "legendFormat": "decoded /s" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "walshadow_config_backfills_pending{mode=\"\"}", + "instant": true, + "legendFormat": "pending", + "refId": "A" + } + ], + "title": "Pending backfills", + "type": "stat" }, { - "refId": "D", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_insertbatch_rows_in_total[$__rate_interval])", - "legendFormat": "\u2192 ingester /s" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "walshadow_uptime_seconds", + "instant": true, + "legendFormat": "uptime", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" }, { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "rate(walshadow_emitter_rows_total[$__rate_interval])", - "legendFormat": "\u2192 ClickHouse /s" - } - ] - }, - { - "id": 22, - "type": "timeseries", - "title": "Cumulative rows: decoded vs in ClickHouse (gap = in pipeline)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 58 - }, - "fieldConfig": { - "defaults": { - "unit": "rows", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "lastNotNull" - ] + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 101, + "panels": [], + "title": "Pipeline", + "type": "row" }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_decoder_decoded_total", - "legendFormat": "decoded (cumulative)" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (route) (rate(walshadow_filter_records_total[$__rate_interval]))", + "legendFormat": "filter {{route}}", + "refId": "A" + }, + { + "expr": "rate(walshadow_decoder_decoded_total[$__rate_interval])", + "legendFormat": "decoded records", + "refId": "B" + } + ], + "title": "WAL records/s", + "type": "timeseries" }, { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_emitter_rows_total", - "legendFormat": "in ClickHouse (cumulative)" - } - ] - }, - { - "id": 23, - "type": "timeseries", - "title": "Per-stage WAL lag (bytes) \u2014 the tail drains here", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 58 - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(walshadow_xacts_committed_total[$__rate_interval])", + "legendFormat": "committed", + "refId": "A" + }, + { + "expr": "rate(walshadow_xacts_aborted_total[$__rate_interval])", + "legendFormat": "aborted", + "refId": "B" + } + ], + "title": "Transactions/s", + "type": "timeseries" }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "clamp_min(walshadow_source_received_lsn - walshadow_filter_lsn, 0)", - "legendFormat": "dispatch lag (received \u2212 filter)" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(walshadow_decode_rows_out_total[$__rate_interval])", + "legendFormat": "decoded", + "refId": "A" + }, + { + "expr": "rate(walshadow_insertbatch_rows_in_total[$__rate_interval])", + "legendFormat": "batched", + "refId": "B" + }, + { + "expr": "rate(walshadow_emitter_rows_total[$__rate_interval])", + "legendFormat": "ClickHouse", + "refId": "C" + } + ], + "title": "Rows/s by stage", + "type": "timeseries" }, { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "clamp_min(walshadow_source_received_lsn - walshadow_decoder_commit_lsn, 0)", - "legendFormat": "decode lag (received \u2212 decoder_commit)" + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 102, + "panels": [], + "title": "Lag and queues", + "type": "row" }, { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)", - "legendFormat": "CH-durable lag (received \u2212 emitter_ack)" - } - ] - }, - { - "id": 27, - "type": "timeseries", - "title": "Pump queue backlog (WAL records waiting)", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 36 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "walshadow_shadow_apply_lag_bytes", + "legendFormat": "source to shadow", + "refId": "A" + }, + { + "expr": "clamp_min(walshadow_source_received_lsn - walshadow_decoder_commit_lsn, 0)", + "legendFormat": "source to decoder", + "refId": "B" + }, + { + "expr": "clamp_min(walshadow_source_received_lsn - walshadow_emitter_ack_lsn, 0)", + "legendFormat": "source to ClickHouse ack", + "refId": "C" + } + ], + "title": "WAL backlog", + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_pump_queue_depth", - "legendFormat": "records waiting for the worker" - } - ] - }, - { - "id": 107, - "type": "row", - "title": "\u2467 switchover", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "panels": [] - }, - { - "id": 30, - "type": "stat", - "title": "Timelines", - "description": "Floor trails source between a fork and the crossing that commits it; the shadow trails both until the crossing is advertised.", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "walshadow_pump_queue_depth", + "legendFormat": "pump records", + "refId": "A" + }, + { + "expr": "clamp_min(walshadow_queue_jobs_out_total - walshadow_decode_jobs_in_total, 0)", + "legendFormat": "decode jobs", + "refId": "B" + }, + { + "expr": "clamp_min(walshadow_decode_rows_out_total - walshadow_insertbatch_rows_in_total, 0)", + "legendFormat": "decoded rows", + "refId": "C" + }, + { + "expr": "clamp_min(walshadow_insertbatch_batches_out_total - walshadow_inserter_batches_in_total, 0)", + "legendFormat": "insert batches", + "refId": "D" + } + ], + "title": "Pipeline queue depth", + "type": "timeseries" }, - "colorMode": "background", - "graphMode": "none", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_source_timeline", - "legendFormat": "source" + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 103, + "panels": [], + "title": "Memory and operations", + "type": "row" }, { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_floor_timeline", - "legendFormat": "floor" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "walshadow_process_resident_memory_bytes", + "legendFormat": "process RSS", + "refId": "A" + }, + { + "expr": "walshadow_resident_payload_bytes", + "legendFormat": "budgeted payload", + "refId": "B" + }, + { + "expr": "walshadow_xact_bytes_in_memory", + "legendFormat": "transaction memory", + "refId": "C" + }, + { + "expr": "walshadow_spill_bytes_active", + "legendFormat": "transaction spill", + "refId": "D" + }, + { + "expr": "walshadow_drain_resident_bytes", + "legendFormat": "commit drain", + "refId": "E" + } + ], + "title": "Memory and spill", + "type": "timeseries" }, { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_shadow_served_timeline", - "legendFormat": "shadow served" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "walshadow_xact_active", + "legendFormat": "active transactions", + "refId": "A" + }, + { + "expr": "walshadow_spill_xacts_active", + "legendFormat": "spilled transactions", + "refId": "B" + }, + { + "expr": "walshadow_config_backfills_pending{mode=\"\"}", + "legendFormat": "pending backfills", + "refId": "C" + }, + { + "expr": "walshadow_config_pending_decl_rels", + "legendFormat": "pending table declarations", + "refId": "D" + }, + { + "expr": "walshadow_crossing_wedged", + "legendFormat": "blocked crossing", + "refId": "E" + }, + { + "expr": "walshadow_source_endpoint_swap_pending", + "legendFormat": "endpoint swap pending", + "refId": "F" + } + ], + "title": "Active work and operator attention", + "type": "timeseries" }, { - "refId": "D", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_shadow_replay_timeline", - "legendFormat": "shadow replay" - } - ] - }, - { - "id": 31, - "type": "timeseries", - "title": "Crossings & prefix bytes verified", - "description": "A fork on a segment boundary repeats no bytes, so zero verified prefix is valid there.", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "custom": { - "lineWidth": 1, - "fillOpacity": 8, - "showPoints": "never" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 104, + "panels": [], + "title": "Source transitions", + "type": "row" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_timeline_switches_total", - "legendFormat": "crossings" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "walshadow_source_timeline", + "legendFormat": "source", + "refId": "A" + }, + { + "expr": "walshadow_floor_timeline", + "legendFormat": "durable floor", + "refId": "B" + }, + { + "expr": "walshadow_shadow_served_timeline", + "legendFormat": "shadow served", + "refId": "C" + }, + { + "expr": "walshadow_shadow_replay_timeline", + "legendFormat": "shadow replay", + "refId": "D" + } + ], + "title": "Timelines", + "type": "timeseries" }, { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_timeline_prefix_bytes_verified_total", - "legendFormat": "prefix bytes verified" + "datasource": { + "type": "prometheus", + "uid": "walshadow-prom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "walshadow_timeline_switches_total", + "legendFormat": "timeline switches", + "refId": "A" + }, + { + "expr": "sum by (reason) (walshadow_timeline_switch_failures_total)", + "legendFormat": "refused: {{reason}}", + "refId": "B" + }, + { + "expr": "walshadow_source_endpoint_swaps_total", + "legendFormat": "endpoint swaps", + "refId": "C" + }, + { + "expr": "walshadow_source_endpoint_swap_failures_total", + "legendFormat": "endpoint swap failures", + "refId": "D" + } + ], + "title": "Transitions and refusals", + "type": "timeseries" } - ] + ], + "refresh": "2s", + "schemaVersion": 39, + "tags": [ + "walshadow", + "cdc", + "quickstart" + ], + "templating": { + "list": [] }, - { - "id": 32, - "type": "timeseries", - "title": "Crossings refused, by reason", - "description": "A refusal the pump parked on keeps the daemon up and serving, so the wedge shows here. Fix what the reason names, then pause and resume.", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "custom": { - "lineWidth": 1, - "fillOpacity": 8, - "showPoints": "never" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_timeline_switch_failures_total", - "legendFormat": "{{reason}}" - } - ] + "time": { + "from": "now-15m", + "to": "now" }, - { - "id": 33, - "type": "stat", - "title": "Source cluster & endpoint swaps", - "description": "System identifier owns every artifact; a repoint onto a different cluster is refused.", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 66 - }, - "fieldConfig": { - "defaults": { - "decimals": 0, - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "background", - "graphMode": "none", - "textMode": "auto", - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_source_info", - "legendFormat": "{{system_id}}" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_source_endpoint_swap_pending", - "legendFormat": "swap pending" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "walshadow-prom" - }, - "expr": "walshadow_source_endpoint_swap_failures_total", - "legendFormat": "swap failures" - } - ] - } - ] + "timepicker": {}, + "timezone": "browser", + "title": "walshadow status", + "uid": "walshadow-status", + "version": 1 } diff --git a/docker/grafana/prometheus.yml b/docker/grafana/prometheus.yml new file mode 100644 index 00000000..eaa8dbd3 --- /dev/null +++ b/docker/grafana/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 2s + evaluation_interval: 2s + +scrape_configs: + - job_name: walshadow + static_configs: + - targets: ["walshadow:9484"] diff --git a/docker/grafana/provisioning/dashboards/dashboards.yml b/docker/grafana/provisioning/dashboards/dashboards.yml index 67741b52..fde0bbcd 100644 --- a/docker/grafana/provisioning/dashboards/dashboards.yml +++ b/docker/grafana/provisioning/dashboards/dashboards.yml @@ -1,4 +1,3 @@ -# Load every dashboard JSON under /var/lib/grafana/dashboards on boot. apiVersion: 1 providers: @@ -6,8 +5,8 @@ providers: orgId: 1 folder: "" type: file - disableDeletion: false - allowUiUpdates: true + disableDeletion: true + allowUiUpdates: false options: path: /var/lib/grafana/dashboards foldersFromFilesStructure: false diff --git a/docker/grafana/provisioning/datasources/datasources.yml b/docker/grafana/provisioning/datasources/datasources.yml index b7a3db5b..26272885 100644 --- a/docker/grafana/provisioning/datasources/datasources.yml +++ b/docker/grafana/provisioning/datasources/datasources.yml @@ -1,6 +1,3 @@ -# Grafana datasources for the walshadow demo. UIDs are pinned so the -# provisioned dashboard (dashboards/walshadow.json) can reference them -# without a UI hookup step. apiVersion: 1 datasources: @@ -10,21 +7,6 @@ datasources: access: proxy url: http://prometheus:9090 isDefault: true + editable: false jsonData: timeInterval: 2s - - # Reads the destination tables directly over CH's native protocol - # (port 9000) so panels can show actual rows landed, not just - # pipeline counters. Provided by GF_INSTALL_PLUGINS on first boot. - - name: ClickHouse - uid: walshadow-ch - type: grafana-clickhouse-datasource - access: proxy - jsonData: - host: clickhouse - port: 9000 - protocol: native - username: default - defaultDatabase: demo - secureJsonData: - password: "" diff --git a/docker/init/clickhouse/01-schema.sql b/docker/init/clickhouse/01-schema.sql deleted file mode 100644 index 3093ca70..00000000 --- a/docker/init/clickhouse/01-schema.sql +++ /dev/null @@ -1,5 +0,0 @@ --- walshadow creates tables, not databases. Pre-create the target database --- so replicate_all can auto-create demo.users (and any other source table) --- into it on first boot. - -CREATE DATABASE IF NOT EXISTS demo; diff --git a/docker/init/clickhouse/02-pgbench.sh b/docker/init/clickhouse/02-pgbench.sh deleted file mode 100755 index 17575896..00000000 --- a/docker/init/clickhouse/02-pgbench.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# Demo-only: pre-create the four pgbench destination tables on CH so the -# walshadow emitter's pinned mappings (ch-config.demo.toml) have targets -# to INSERT into. No-op unless WALSHADOW_DEMO_PGBENCH is set. Column -# order + synthetic _lsn/_xid/_commit_ts/_is_deleted trailer mirror the -# emitter's TablePlan; engine ReplacingMergeTree(_lsn, _is_deleted) drops -# deletes on FINAL. Shapes match tests/pgbench_acceptance.rs. - -set -euo pipefail - -[ -n "${WALSHADOW_DEMO_PGBENCH:-}" ] || exit 0 - -clickhouse-client -n --query " -CREATE DATABASE IF NOT EXISTS demo; - -CREATE TABLE IF NOT EXISTS demo.pgbench_accounts ( - aid Int32, bid Int32, abalance Int32, filler String, - _lsn UInt64, _xid UInt32, - _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool -) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY aid; - -CREATE TABLE IF NOT EXISTS demo.pgbench_branches ( - bid Int32, bbalance Int32, filler Nullable(String), - _lsn UInt64, _xid UInt32, - _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool -) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY bid; - -CREATE TABLE IF NOT EXISTS demo.pgbench_tellers ( - tid Int32, bid Int32, tbalance Int32, filler Nullable(String), - _lsn UInt64, _xid UInt32, - _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool -) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY tid; - -CREATE TABLE IF NOT EXISTS demo.pgbench_history ( - tid Int32, bid Int32, aid Int32, delta Int32, - mtime DateTime64(6), filler Nullable(String), - _lsn UInt64, _xid UInt32, - _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool -) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY (tid, mtime, aid); -" - -echo "walshadow-demo: pgbench destination tables created on ClickHouse" diff --git a/docker/init/source/00-hba.sh b/docker/init/source/00-hba.sh deleted file mode 100755 index 24b78ec0..00000000 --- a/docker/init/source/00-hba.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -# postgres image's POSTGRES_HOST_AUTH_METHOD only writes a `host all all` -# rule, replication needs its own line. Init scripts run before the -# server restarts into normal mode, so this takes effect on restart. - -set -euo pipefail -echo "host replication all all trust" >> "$PGDATA/pg_hba.conf" diff --git a/docker/init/source/01-schema.sql b/docker/init/source/01-schema.sql deleted file mode 100644 index 04e4e78c..00000000 --- a/docker/init/source/01-schema.sql +++ /dev/null @@ -1,18 +0,0 @@ --- walshadow demo source schema. Three rows naming the parties in any --- relation of labour: workman, master, state. REPLICA IDENTITY FULL --- ships the old-tuple image on UPDATE/DELETE so each change is --- observable on the wire, not inferred. - -CREATE SCHEMA IF NOT EXISTS demo; - -CREATE TABLE demo.users ( - id bigint PRIMARY KEY, - name text NOT NULL, - email text NOT NULL -); -ALTER TABLE demo.users REPLICA IDENTITY FULL; - -INSERT INTO demo.users (id, name, email) VALUES - (1, 'Opifex', 'opifex@rerum.novarum'), - (2, 'Dominus', 'dominus@rerum.novarum'), - (3, 'Respublica', 'respublica@rerum.novarum'); diff --git a/docker/init/source/02-pgbench.sh b/docker/init/source/02-pgbench.sh deleted file mode 100755 index a737e618..00000000 --- a/docker/init/source/02-pgbench.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Demo-only: seed the standard pgbench TPC-B schema so the `pgbench` -# hammer service has tables to pound. No-op unless WALSHADOW_DEMO_PGBENCH -# is set (the lean base stack leaves it unset, keeping this an empty -# init step). Runs inside the postgres image's init phase, so the four -# tables land in source's data dir before walshadow takes its base -# backup — they're present at bootstrap, satisfying preflight's -# "mapped relation exists with REPLICA IDENTITY FULL" gate. - -set -euo pipefail - -[ -n "${WALSHADOW_DEMO_PGBENCH:-}" ] || exit 0 - -SCALE="${PGBENCH_SCALE:-1}" - -# `-i` drops+recreates pgbench_{accounts,branches,tellers,history} and -# loads scale*100k accounts. Quiet the per-100k progress chatter. -pgbench -i -s "$SCALE" -q -U "$POSTGRES_USER" -d "$POSTGRES_DB" - -# walshadow decodes physical WAL; UPDATE/DELETE need the full old-tuple -# image on the wire, which only REPLICA IDENTITY FULL ships. Preflight -# refuses to stream a mapped relation without it. -psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' -ALTER TABLE pgbench_accounts REPLICA IDENTITY FULL; -ALTER TABLE pgbench_branches REPLICA IDENTITY FULL; -ALTER TABLE pgbench_tellers REPLICA IDENTITY FULL; -ALTER TABLE pgbench_history REPLICA IDENTITY FULL; -SQL - -echo "walshadow-demo: pgbench schema seeded (scale=$SCALE), REPLICA IDENTITY FULL set" diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml deleted file mode 100644 index 51c5d7c2..00000000 --- a/docker/prometheus/prometheus.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Prometheus scrape config for the walshadow demo. Two targets: -# walshadow — the daemon's hand-rolled /metrics surface -# (src/metrics.rs), updated every status tick -# (WALSHADOW_STATUS_INTERVAL=1s in the demo). -# postgres-exporter — source PG's pg_stat_* (TPS, tuple write rates). -# ClickHouse is read directly by Grafana via the native datasource, not -# scraped here. - -global: - scrape_interval: 2s - evaluation_interval: 2s - -scrape_configs: - - job_name: walshadow - static_configs: - - targets: ["walshadow:9484"] - - - job_name: postgres - static_configs: - - targets: ["postgres-exporter:9187"] diff --git a/plans/control.md b/plans/control.md index ddfd2c6f..8f040a6b 100644 --- a/plans/control.md +++ b/plans/control.md @@ -15,13 +15,14 @@ modeled on `metrics::serve` + the `shadow_stream.rs` bind pattern). Absent → disabled. The client is the same binary: `walshadow-stream ctl ` with the command body as a TOML fragment on stdin (`ctl apply < `, `pause`, `source `, …) build their own body +in `src/ops/ctl.rs`, above these verbs rather than part of them. The command is a bare **verb**. The TOML body already has section headers, so target rides in body, not command: `apply` takes an arbitrary fragment (any mix of `[source]` / `[ch]` / `[table.*]` / `[stream]`) and merges it in one atomic -reload. CLI ergonomics (friendly aliases) are a separate layer above these verbs, -not the daemon's concern. +reload. Wire protocol (one request per connection, `handle_conn`): a `` header line selects the handler; everything after the first newline is the config, a @@ -66,12 +67,12 @@ that is *also* TOML (`show`/`status` are tables, `tables`/`columns` are endpoint while paused, which is the promotion target once step 4's repoint lands ([failover.md](failover.md) §Operator protocol). - `tables` (`namespace`) — enumerate source `pg_class` as an `[[tables]]` array - (`namespace`, `name`, `selected`, `replica_identity`), marking `selected` from - the merged config; `schemas` (array of strings), `columns` (`namespace`, + (`namespace`, `name`, `selected`, `replica_identity`, `has_row_key`), marking + `selected` from the merged config; `schemas` (array of strings), `columns` (`namespace`, `relname` → `[[columns]]` with `name`, `type`, `notnull`) — source-PG introspection. -`SharedCtx { ch_config, source_base, metrics, reloader, frag_lock }` is handed to +`SharedCtx { ch_config, cli_base, metrics, reloader, frag_lock }` is handed to the handlers; `frag_lock` serializes fragment read-modify-write so concurrent `apply`/`unset` can't lose an update or race a rollback. `Reloader` holds only the running session's `Arc` (`set_resolver`, `reload`) — there is @@ -83,8 +84,10 @@ Config is the daemon's own TOML, `--ch-config` **plus** every `*.toml` in the sibling `.d/` directory (e.g. `ch-config.toml` → `ch-config.d/`), deep-merged in lexical filename order — Postgres `include_dir` style (`ch_emitter::load_merged` / `merge_tables`). `load_effective(path, base)` layers -the CLI-arg `[source]` defaults *under* the file so source connection resolves -file-over-CLI (matches `EmitterConfig` boot). +the CLI-arg `[source]` / `[ch]` defaults *under* the file so connections resolve +file-over-CLI (matches `EmitterConfig` boot). That base is `cli_base(args)`: the +discrete flags with `--source-url` / `--ch-url` merged over them, which is how a +URL pair alone configures a daemon with no config file. The control API writes **only its own fragment**, `ch-config.d/50-api.toml` (`frag_path`) — sparse, only the keys `apply`/`unset` set. The operator's base @@ -216,6 +219,9 @@ read-only mount. ## Files - `src/ops/control.rs` — socket, protocol, handlers, `Reloader`, `SharedCtx`. +- `src/ops/ctl.rs` — client-side verb sugar + reply rendering. +- `src/ops/introspect.rs` — the `tables` / `schemas` / `columns` catalog reads, + shared with `init`. - `src/bin/stream.rs` — `--control-socket`, `ctl` subcommand, `run`/`run_session`, `cli_source_base`, `spawn_sighup_reload`, pump `paused` gate + endpoint swap (`resume_source_feed`, `SOURCE_SWAP_RETRY`). @@ -243,7 +249,9 @@ read-only mount. threading the watch through the store and clearing its created-table set on swap; off by default, so it is a known gap rather than a broken path. - Control still opens source-PG read connections for introspection - (`// TODO` on `pg_connect`) — route through the daemon's catalog later. + (`// TODO` on `pg_connect`) — route through the daemon's catalog later. Those + connections go through `SourceConn` + `source_feed::open_sql_client`, so they + honour `sslmode` and unix-socket hosts like the daemon's own. - The `walshadow-peerdb` shim's PAUSED/RUNNING map to `apply [stream] paused`; create-mirror maps to one `apply` carrying `[source]` + `[ch]` + `[table.*]` (source, dest, and tables in a single atomic reload). Repointing a live diff --git a/src/bin/stream.rs b/src/bin/stream.rs index f2bdc852..cecd40aa 100644 --- a/src/bin/stream.rs +++ b/src/bin/stream.rs @@ -265,23 +265,61 @@ impl RecordSink for DaemonSinks { } } -/// `walshadow-stream ctl `: drive a running daemon's control socket. -/// Detected before daemon-arg parsing so `ctl` needn't supply daemon args. +/// `walshadow-stream init`: write a config from two connection URLs, so a +/// first run needs no TOML. Detected before daemon-arg parsing, same as `ctl`. #[derive(Debug, Parser)] #[command( - name = "walshadow-stream ctl", - about = "Control a running walshadow-stream daemon." + name = "walshadow-stream init", + about = "Probe source + destination, pick tables, write the config." )] -struct CtlArgs { +struct InitArgs { + /// Config to write; the daemon then runs with `--ch-config ` #[arg( long, - env = "WALSHADOW_CONTROL_SOCKET", - default_value = "/run/walshadow/control.sock" + env = "WALSHADOW_CH_CONFIG", + default_value = "/etc/walshadow/ch-config.toml" )] - socket: PathBuf, - /// Control verb, such as `status` or `apply`, read TOML body from stdin - #[arg(trailing_var_arg = true, required = true)] - request: Vec, + config: PathBuf, + #[arg(long, env = walshadow::init::SOURCE_URL_ENV)] + source_url: Option, + #[arg(long, env = walshadow::init::CH_URL_ENV)] + ch_url: Option, + /// Replicate this table. Two words, schema then table; repeat per table + #[arg(long, num_args = 2, value_names = ["SCHEMA", "TABLE"])] + table: Vec, + /// Replicate every table that has a row key + #[arg(long)] + all_tables: bool, + /// Restrict listing (and `--all-tables`) to one schema + #[arg(long)] + schema: Option, + /// Backfill of rows that pre-date the opt-in + #[arg(long, default_value = "copy")] + initial_load: String, + /// Overwrite an existing config + #[arg(long)] + force: bool, +} + +impl InitArgs { + fn into_opts(self) -> walshadow::init::InitOpts { + walshadow::init::InitOpts { + config: self.config, + source_url: self.source_url, + ch_url: self.ch_url, + tables: self + .table + .as_chunks::<2>() + .0 + .iter() + .map(|pair| RelName::new(&pair[0], &pair[1])) + .collect(), + all_tables: self.all_tables, + namespace: self.schema, + initial_load: self.initial_load, + force: self.force, + } + } } #[derive(Debug, Parser)] @@ -290,6 +328,21 @@ struct CtlArgs { about = "Stream + filter physical WAL from source PG." )] struct Args { + /// Source connection as one URL, eg + /// `postgres://user:password@host:5432/dbname?sslmode=require`. Wins + /// over the discrete `--host` / `--port` / … flags, loses to + /// `[source]` in `--ch-config`, same as they do + #[arg(long, env = walshadow::init::SOURCE_URL_ENV)] + source_url: Option, + /// Destination as one URL, eg + /// `clickhouse://user:password@host:9000/database`. Supplies `[ch]` + /// when no config file does, which is what turns the emitter on + #[arg(long, env = walshadow::init::CH_URL_ENV)] + ch_url: Option, + /// `[source]` / `[ch]` decoded from the two URL flags once at startup, + /// then merged over the discrete flags by [`cli_base`] + #[arg(skip)] + url_base: toml::Table, /// Source PG host (TCP) or unix socket directory (leading `/`) #[arg(long, default_value = "localhost")] host: String, @@ -532,8 +585,14 @@ async fn main() -> Result<()> { let argv: Vec = std::env::args().collect(); if argv.get(1).map(String::as_str) == Some("ctl") { let rest = std::iter::once(format!("{} ctl", argv[0])).chain(argv.into_iter().skip(2)); - let ctl = CtlArgs::parse_from(rest); - return run_ctl(ctl.socket, ctl.request).await; + let (socket, command) = walshadow::ctl::Cli::parse_from(rest).into_parts()?; + return run_ctl(&socket, command).await; + } + if argv.get(1).map(String::as_str) == Some("init") { + let rest = std::iter::once(format!("{} init", argv[0])).chain(argv.into_iter().skip(2)); + let opts = InitArgs::parse_from(rest).into_opts(); + init_tracing(None); + return walshadow::init::run(opts).await; } let args = Args::parse(); walshadow::trace::set_sample_ratio(args.trace_sample_ratio); @@ -555,20 +614,19 @@ async fn main() -> Result<()> { result } -async fn run_ctl(socket: PathBuf, request: Vec) -> Result<()> { +async fn run_ctl(socket: &Path, cmd: walshadow::ctl::Command) -> Result<()> { use std::io::{IsTerminal, Read}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let verb = request.first().map(String::as_str).unwrap_or_default(); - let config: toml::Table = if std::io::stdin().is_terminal() { - toml::Table::new() + let body = if cmd.reads_stdin && !std::io::stdin().is_terminal() { + let mut raw = String::new(); + std::io::stdin().read_to_string(&mut raw)?; + raw.parse().context("parse config body as TOML")? } else { - let mut body = String::new(); - std::io::stdin().read_to_string(&mut body)?; - body.parse().context("parse config body as TOML")? + cmd.body }; - let doc = walshadow::control::encode_request(verb, config)?; - let mut stream = tokio::net::UnixStream::connect(&socket) + let doc = walshadow::control::encode_request(&cmd.verb, body)?; + let mut stream = tokio::net::UnixStream::connect(socket) .await .with_context(|| format!("connect control socket {}", socket.display()))?; stream.write_all(doc.as_bytes()).await?; @@ -576,20 +634,19 @@ async fn run_ctl(socket: PathBuf, request: Vec) -> Result<()> { stream.shutdown().await.ok(); let mut resp = String::new(); stream.read_to_string(&mut resp).await?; - let first = resp.lines().next().unwrap_or(""); - if let Some(rest) = first.strip_prefix("OK") { - let rest = rest.trim(); - if !rest.is_empty() { - println!("{rest}"); - } - for l in resp.lines().skip(1) { - println!("{l}"); - } - Ok(()) - } else { + let (head, payload) = resp.split_once('\n').unwrap_or((resp.as_str(), "")); + let Some(trailer) = head.strip_prefix("OK") else { eprint!("{resp}"); std::process::exit(1); + }; + if !trailer.trim().is_empty() { + println!("{}", trailer.trim()); + } + let rendered = walshadow::ctl::render(&cmd.verb, payload); + if !rendered.trim().is_empty() { + println!("{}", rendered.trim_end()); } + Ok(()) } /// OTLP/gRPC batch tracer provider for `endpoint`. Must run inside the tokio @@ -673,9 +730,11 @@ fn init_tracing( provider } -/// `[source]` defaults from the CLI args — the base layer under the config file -/// for connection resolution, shared by the session and the control surface. -fn cli_source_base(args: &Args) -> toml::Table { +/// `[source]` + `[ch]` defaults from the CLI args — the base layer under the +/// config file for connection resolution, shared by the session and the +/// control surface. A `--source-url` / `--ch-url` merges over the discrete +/// flags, so the URL wins wherever both name a field. +fn cli_base(args: &Args) -> toml::Table { let mut s = toml::Table::new(); s.insert("host".into(), args.host.clone().into()); s.insert("port".into(), (args.port as i64).into()); @@ -687,9 +746,32 @@ fn cli_source_base(args: &Args) -> toml::Table { s.insert("sslmode".into(), args.sslmode.clone().into()); let mut root = toml::Table::new(); root.insert("source".into(), toml::Value::Table(s)); + walshadow::ch_emitter::merge_tables(&mut root, args.url_base.clone()); root } +/// Decode `--source-url` / `--ch-url` once, so every later `cli_base` is a +/// pure merge and a malformed URL fails at startup rather than mid-reload +fn url_base(args: &Args) -> Result { + let mut root = toml::Table::new(); + // An env var exported empty reads as unset, so a compose file may pass + // the name through unconditionally + let nonempty = |u: &&String| !u.trim().is_empty(); + if let Some(url) = args.source_url.as_ref().filter(nonempty) { + root.insert( + "source".into(), + toml::Value::Table(walshadow::dsn::source_table(url)?), + ); + } + if let Some(url) = args.ch_url.as_ref().filter(nonempty) { + root.insert( + "ch".into(), + toml::Value::Table(walshadow::dsn::ch_table(url)?), + ); + } + Ok(root) +} + fn spawn_sighup_reload( mut sig: tokio::signal::unix::Signal, reloader: Arc, @@ -722,9 +804,10 @@ fn validate_transport_args(args: &Args) -> Result<()> { /// Process-lifetime entry: bind metrics + control socket + SIGHUP, then stream /// one session. Every reconfigure (socket / SIGHUP) is a live reload — no /// restart. Ctrl-C breaks the pump loop and drains gracefully. -async fn run(args: Args) -> Result<()> { +async fn run(mut args: Args) -> Result<()> { use walshadow::control::{Reloader, SharedCtx}; + args.url_base = url_base(&args)?; validate_transport_args(&args)?; let sighup = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup()) .inspect_err(|e| { @@ -764,7 +847,7 @@ async fn run(args: Args) -> Result<()> { .context("--control-socket requires --ch-config")?; let ctx = SharedCtx { ch_config, - source_base: cli_source_base(&args), + cli_base: cli_base(&args), metrics: metrics.clone(), reloader: reloader.clone(), frag_lock: Arc::new(Mutex::new(())), @@ -792,10 +875,10 @@ async fn run_session( let metrics = metrics.clone(); let merged: toml::Table = match args.ch_config.as_deref() { - Some(p) => walshadow::ch_emitter::load_effective(p, cli_source_base(args)) + Some(p) => walshadow::ch_emitter::load_effective(p, cli_base(args)) .await .with_context(|| format!("load config {}", p.display()))?, - None => cli_source_base(args), + None => cli_base(args), }; // Applied source endpoint. Boot resolves it file-over-CLI; a later reload // republishes it on the config watch and the pump swaps its feed. @@ -840,6 +923,14 @@ async fn run_session( } else { None }; + // Before anything dials CH naming that database in its handshake — the + // bootstrap insert tail is first, and its failure there reads as a + // bootstrap fault rather than a missing destination + if let Some(cfg) = ch_config.as_ref() { + walshadow::ch_ddl::ensure_boot_database(cfg) + .await + .with_context(|| format!("reach ClickHouse {}:{}", cfg.host, cfg.port))?; + } // QueueingRecordSink knobs feed both the CH and metrics-only pipelines, // so resolve here while `ch_config` is still in scope (it is consumed // into `emitter_cfg` below). CLI over `[ch]` over the built-in default. @@ -1461,7 +1552,7 @@ async fn run_session( &emitter_cfg, cli_overrides, args.ch_config.clone(), - cli_source_base(args), + cli_base(args), mapping.clone(), ); reloader.set_resolver(Some(resolver.clone())).await; @@ -1516,44 +1607,28 @@ async fn run_session( emitter_stats_handle = Some(stats.clone()); // Backfiller for `initial_load` opt-ins (COPY / backup-sourced): // own source session + CH tail per backfill or pass, spill-dir - // ledger dedups restarts. - let toml_initial_load = emitter_cfg - .table_initial_loads - .values() - .chain( - emitter_cfg - .table_opt_ins - .values() - .filter_map(|r| r.initial_load.as_ref()), - ) - .chain( - emitter_cfg - .table_entries - .iter() - .filter_map(|(_, _, rule)| rule.initial_load.as_ref()), - ) - .any(|mode| InitialLoadMode::parse(mode).is_some_and(|m| m != InitialLoadMode::None)); + // ledger dedups restarts. Wired whenever the emitter runs, since an + // opt-in arriving later over the control socket or the overlay would + // otherwise silently skip its backfill; idle it costs one ledger read. // One validated resident-payload pool for the pipeline and every // concurrent backup pass let pipeline_budget = walshadow::pipeline::build_budget(&emitter_cfg, emitter_cfg.decoder_pool_size) .map_err(|e| anyhow::anyhow!("memory budget: {e}"))?; - if emitter_cfg.runtime_config_schema.is_some() || toml_initial_load { - copy_backfiller = Some(Arc::new( - walshadow::copy_backfill::CopyBackfiller::new( - cfg.clone(), - emitter_cfg.clone(), - mapping.clone(), - stats.clone(), - catalog.clone(), - desc_log.clone(), - &args.spill_dir, - Some(config_rx.clone()), - Some(pipeline_budget.clone()), - ) - .await, - )); - } + copy_backfiller = Some(Arc::new( + walshadow::copy_backfill::CopyBackfiller::new( + cfg.clone(), + emitter_cfg.clone(), + mapping.clone(), + stats.clone(), + catalog.clone(), + desc_log.clone(), + &args.spill_dir, + Some(config_rx.clone()), + Some(pipeline_budget.clone()), + ) + .await, + )); let backfiller_effects: Option> = copy_backfiller.clone().map(|backfiller| backfiller as _); // Re-materialise per-table opt-in scope from the seeded config_table @@ -3747,7 +3822,7 @@ async fn connect_source_waiting( let Some(path) = args.ch_config.as_deref() else { continue; }; - match walshadow::ch_emitter::load_effective(path, cli_source_base(args)).await { + match walshadow::ch_emitter::load_effective(path, cli_base(args)).await { Ok(table) => match SourceConn::from_table(&table).map(|mut next| { // Preserve CLI slot override across reloads if args.slot.is_some() { @@ -4359,7 +4434,7 @@ async fn bootstrap_build_mapping( emitter_cfg, cli_overrides, args.ch_config.clone(), - cli_source_base(args), + cli_base(args), mapping.clone(), ); let (ddl_cfg, merged_tables) = { diff --git a/src/config.rs b/src/config.rs index 76e78e75..4a644c18 100644 --- a/src/config.rs +++ b/src/config.rs @@ -311,7 +311,7 @@ pub struct ConfigResolver { toml_path: Option, /// CLI-arg `[source]` base layer, merged under the file on reload (matches /// boot's `load_effective`). - cli_source_base: toml::Table, + cli_base: toml::Table, cli: CliOverrides, inner: Mutex, tx: watch::Sender>, @@ -339,7 +339,7 @@ impl ConfigResolver { base: &EmitterConfig, cli: CliOverrides, toml_path: Option, - cli_source_base: toml::Table, + cli_base: toml::Table, mapping: MappingHandle, ) -> (Arc, watch::Receiver>) { let overlay = ConfigOverlay::default(); @@ -348,7 +348,7 @@ impl ConfigResolver { let (tx, rx) = watch::channel(Arc::new(initial)); let this = Arc::new(Self { toml_path, - cli_source_base, + cli_base, cli, inner: Mutex::new(MergeInputs { base: base.clone(), @@ -818,7 +818,7 @@ impl ConfigResolver { let Some(path) = &self.toml_path else { return Ok(()); }; - let merged = crate::ch_emitter::load_effective(path, self.cli_source_base.clone()).await?; + let merged = crate::ch_emitter::load_effective(path, self.cli_base.clone()).await?; let base = EmitterConfig::from_table(&merged)?; let mut inner = self.inner.lock().await; inner.base = base; diff --git a/src/dsn.rs b/src/dsn.rs new file mode 100644 index 00000000..a57db83b --- /dev/null +++ b/src/dsn.rs @@ -0,0 +1,445 @@ +//! Connection URLs → config TOML sections +//! +//! `postgres://` and `clickhouse://` strings are what a hosted provider +//! hands an operator, so they are accepted wherever `[source]` / `[ch]` +//! are configured, with no transcription of fields into TOML. Output is a +//! `[source]` / `[ch]` table, merged like any other config layer +//! ([`crate::ch_emitter::load_effective`]) + +use percent_encoding::percent_decode_str; +use thiserror::Error; +use tokio_postgres::config::{Host, SslMode}; +use toml::{Table, Value}; +use url::{ParseError, Url}; + +#[derive(Debug, Error)] +pub enum DsnError { + #[error("{url:?}: expected a {expected} URL")] + Scheme { url: String, expected: &'static str }, + #[error("{url:?}: invalid {kind} URL: {reason}")] + Invalid { + url: String, + kind: &'static str, + reason: String, + }, + #[error("{0:?}: no host")] + NoHost(String), + #[error("{0:?}: port not a number in 1..=65535")] + BadPort(String), + #[error("{url:?}: unknown parameter {key:?} (supported: {supported})")] + UnknownParam { + url: String, + key: String, + supported: &'static str, + }, + #[error("{url:?}: parameter {key:?} takes true or false, got {got:?}")] + NotBool { + url: String, + key: String, + got: String, + }, + #[error("{0:?}: URL component is not UTF-8")] + BadEncoding(String), +} + +const PG_PARAMS: &str = "sslmode, slot, host, port, user, password, dbname"; +const CH_PARAMS: &str = "secure, compression, database, user, password, port"; + +/// `postgres://user:pass@host:5432/dbname?sslmode=require&slot=walshadow` +/// +/// Unix sockets ride the libpq spelling: `postgres:///dbname?host=/run/postgresql` +pub fn source_table(url: &str) -> Result { + // `user@` with an empty host is a libpq unix socket, which the URL crate + // refuses, so scheme and query come apart by hand here + let (base, query) = url.split_once('?').unwrap_or((url, "")); + let scheme = base.split_once("://").map_or("", |(s, _)| s); + if !["postgres", "postgresql"].contains(&scheme.to_ascii_lowercase().as_str()) { + return Err(DsnError::Scheme { + url: url.into(), + expected: "postgres://", + }); + } + if let Some(port) = authority_port(base) { + parse_port(url, port)?; + } + let mut slot = None; + let mut host_set = false; + let mut port_set = false; + let mut sslmode_set = false; + let mut retained = Vec::new(); + for pair in query.split('&').filter(|s| !s.is_empty()) { + let (raw_key, raw_value) = pair.split_once('=').unwrap_or((pair, "")); + let key = decode(url, raw_key)?.to_ascii_lowercase(); + if key == "slot" { + slot = Some(decode(url, raw_value)?); + } else if matches!( + key.as_str(), + "sslmode" | "host" | "port" | "user" | "password" | "dbname" + ) { + host_set |= key == "host"; + port_set |= key == "port"; + sslmode_set |= key == "sslmode"; + retained.push(format!("{key}={raw_value}")); + } else { + return Err(DsnError::UnknownParam { + url: url.into(), + key, + supported: PG_PARAMS, + }); + } + } + let dsn = if retained.is_empty() { + base.into() + } else { + format!("{base}?{}", retained.join("&")) + }; + let config = dsn + .parse::() + .map_err(|e| invalid(url, "PostgreSQL", pg_reason(&e)))?; + let mut out = Table::new(); + if let Some(v) = config.get_user() { + out.insert("user".into(), v.into()); + } + if let Some(v) = config.get_password() { + let v = std::str::from_utf8(v).map_err(|e| invalid(url, "PostgreSQL", e))?; + out.insert("password".into(), v.into()); + } + if let Some(v) = config.get_dbname() { + out.insert("dbname".into(), v.into()); + } + if let Some(v) = slot { + out.insert("slot".into(), v.into()); + } + let sslmode = match config.get_ssl_mode() { + SslMode::Disable => "disable", + SslMode::Prefer => "prefer", + SslMode::Require => "require", + mode => { + return Err(invalid( + url, + "PostgreSQL", + format!("unsupported sslmode {mode:?}"), + )); + } + }; + if sslmode_set { + out.insert("sslmode".into(), sslmode.into()); + } + + let hosts = config.get_hosts(); + let host = if host_set { + hosts.last() + } else { + match hosts { + [host] => Some(host), + [] => None, + _ => { + return Err(invalid( + url, + "PostgreSQL", + "multiple hosts are not supported", + )); + } + } + }; + let host = match host { + Some(Host::Tcp(host)) => host.clone(), + #[cfg(unix)] + Some(Host::Unix(host)) => host + .to_str() + .ok_or_else(|| invalid(url, "PostgreSQL", "host is not UTF-8"))? + .into(), + None => return Err(DsnError::NoHost(url.into())), + }; + out.insert("host".into(), host.into()); + let ports = config.get_ports(); + let port = match (port_set, ports) { + (true, [.., port]) | (false, [port]) if *port != 0 => *port, + (false, []) => 5432, + _ => return Err(DsnError::BadPort(url.into())), + }; + out.insert("port".into(), i64::from(port).into()); + Ok(out) +} + +/// `clickhouse://user:pass@host:9000/database?compression=lz4` +/// +/// `clickhouses://` is the same with TLS, matching `[ch] secure = true` +pub fn ch_table(url: &str) -> Result { + let parsed = parse_url(url, &["clickhouse", "clickhouses"], "clickhouse://")?; + let secure = parsed.scheme() == "clickhouses"; + let mut out = Table::new(); + let mut port = parsed + .port() + .map(|port| { + (port != 0) + .then_some(port) + .ok_or_else(|| DsnError::BadPort(url.into())) + }) + .transpose()?; + out.insert("secure".into(), secure.into()); + if !parsed.username().is_empty() { + let v = decode(url, parsed.username())?; + out.insert("user".into(), v.into()); + } + if let Some(v) = parsed.password() { + let v = decode(url, v)?; + out.insert("password".into(), v.into()); + } + if let Some(v) = parsed.path().strip_prefix('/').filter(|s| !s.is_empty()) { + let v = decode(url, v)?; + out.insert("database".into(), v.into()); + } + for pair in parsed + .query() + .unwrap_or_default() + .split('&') + .filter(|s| !s.is_empty()) + { + let (raw_key, raw_value) = pair.split_once('=').unwrap_or((pair, "")); + let k = decode(url, raw_key)?.to_ascii_lowercase(); + let v = decode(url, raw_value)?; + match k.as_str() { + "compression" | "database" | "user" | "password" => { + out.insert(k, v.into()); + } + "secure" => { + out.insert("secure".into(), parse_bool(url, &k, &v)?.into()); + } + "port" => port = Some(parse_port(url, &v)?), + _ => { + return Err(DsnError::UnknownParam { + url: url.into(), + key: k, + supported: CH_PARAMS, + }); + } + } + } + out.insert( + "host".into(), + parsed + .host_str() + .ok_or_else(|| DsnError::NoHost(url.into()))? + .into(), + ); + // 9440 is the CH-Native TLS port, 9000 the plaintext one + let default_port = if secure_value(&out) { 9440 } else { 9000 }; + out.insert( + "port".into(), + i64::from(port.unwrap_or(default_port)).into(), + ); + Ok(out) +} + +fn secure_value(t: &Table) -> bool { + t.get("secure").and_then(Value::as_bool).unwrap_or(false) +} + +fn parse_url(url: &str, schemes: &[&str], expected: &'static str) -> Result { + let parsed = Url::parse(url).map_err(|e| match e { + ParseError::InvalidPort => DsnError::BadPort(url.into()), + _ => invalid(url, "connection", e), + })?; + if !schemes.contains(&parsed.scheme()) { + return Err(DsnError::Scheme { + url: url.into(), + expected, + }); + } + Ok(parsed) +} + +/// Port text of `scheme://[user[:pass]@]host[:port]`, IPv6 brackets aside +fn authority_port(base: &str) -> Option<&str> { + let authority = base.split_once("://")?.1.split('/').next()?; + let hostport = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + let tail = hostport.rsplit_once(']').map_or(hostport, |(_, t)| t); + tail.rsplit_once(':').map(|(_, port)| port) +} + +/// tokio-postgres keeps the readable half of a parse failure in `source` +fn pg_reason(e: &tokio_postgres::Error) -> String { + std::error::Error::source(e).map_or_else(|| e.to_string(), |src| format!("{e}: {src}")) +} + +fn parse_port(url: &str, raw: &str) -> Result { + raw.parse::() + .ok() + .filter(|p| *p != 0) + .ok_or_else(|| DsnError::BadPort(url.into())) +} + +fn parse_bool(url: &str, key: &str, raw: &str) -> Result { + match raw.to_ascii_lowercase().as_str() { + "true" | "1" | "yes" | "on" => Ok(true), + "false" | "0" | "no" | "off" => Ok(false), + _ => Err(DsnError::NotBool { + url: url.into(), + key: key.into(), + got: raw.into(), + }), + } +} + +fn decode(url: &str, value: &str) -> Result { + percent_decode_str(value) + .decode_utf8() + .map(String::from) + .map_err(|_| DsnError::BadEncoding(url.into())) +} + +fn invalid(url: &str, kind: &'static str, reason: impl std::fmt::Display) -> DsnError { + DsnError::Invalid { + url: url.into(), + kind, + reason: reason.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(t: &Table, k: &str) -> String { + t.get(k) + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("{k} missing")) + .into() + } + fn i(t: &Table, k: &str) -> i64 { + t.get(k) + .and_then(Value::as_integer) + .unwrap_or_else(|| panic!("{k} missing")) + } + + #[test] + fn source_url_full() { + let t = source_table("postgres://repl:s3cret@db.example:5433/app?sslmode=require&slot=ws") + .unwrap(); + assert_eq!(s(&t, "host"), "db.example"); + assert_eq!(i(&t, "port"), 5433); + assert_eq!(s(&t, "user"), "repl"); + assert_eq!(s(&t, "password"), "s3cret"); + assert_eq!(s(&t, "dbname"), "app"); + assert_eq!(s(&t, "sslmode"), "require"); + assert_eq!(s(&t, "slot"), "ws"); + } + + #[test] + fn source_url_defaults_port_and_omits_unset_keys() { + let t = source_table("postgresql://db.example/app").unwrap(); + assert_eq!(i(&t, "port"), 5432); + assert!(!t.contains_key("user")); + assert!(!t.contains_key("password")); + assert!(!t.contains_key("sslmode")); + } + + #[test] + fn source_url_unix_socket_via_host_param() { + let t = source_table("postgres:///app?host=%2Fvar%2Frun%2Fpostgresql").unwrap(); + assert_eq!(s(&t, "host"), "/var/run/postgresql"); + assert_eq!(s(&t, "dbname"), "app"); + } + + #[test] + fn source_url_unix_socket_keeps_credentials() { + let t = source_table("postgres://repl@/app?host=%2Fvar%2Frun%2Fpostgresql").unwrap(); + assert_eq!(s(&t, "host"), "/var/run/postgresql"); + assert_eq!(s(&t, "user"), "repl"); + assert_eq!(s(&t, "dbname"), "app"); + } + + #[test] + fn source_url_query_endpoint_overrides_authority() { + let t = source_table("postgres://old:5432/app?HOST=new&PORT=5433").unwrap(); + assert_eq!(s(&t, "host"), "new"); + assert_eq!(i(&t, "port"), 5433); + } + + #[test] + fn source_url_password_holds_reserved_chars() { + let t = source_table("postgres://u:p%40ss%3Aword@h/d").unwrap(); + assert_eq!(s(&t, "password"), "p@ss:word"); + assert_eq!(s(&t, "user"), "u"); + assert_eq!(s(&t, "host"), "h"); + } + + #[test] + fn source_url_ipv6_literal() { + let t = source_table("postgres://u@[::1]:5433/d").unwrap(); + assert_eq!(s(&t, "host"), "::1"); + assert_eq!(i(&t, "port"), 5433); + } + + #[test] + fn source_url_rejects_unknown_param() { + let e = source_table("postgres://h/d?sslcert=x").unwrap_err(); + assert!(matches!(e, DsnError::UnknownParam { .. }), "{e}"); + } + + #[test] + fn source_url_uses_postgres_validation() { + let e = source_table("postgres://h/d?sslmode=maybe").unwrap_err(); + assert!(matches!(e, DsnError::Invalid { .. }), "{e}"); + } + + #[test] + fn source_url_rejects_other_scheme() { + assert!(matches!( + source_table("mysql://h/d").unwrap_err(), + DsnError::Scheme { .. } + )); + } + + #[test] + fn ch_url_plain_defaults_native_port() { + let t = ch_table("clickhouse://default@ch.example/cdc").unwrap(); + assert_eq!(i(&t, "port"), 9000); + assert_eq!(t.get("secure").and_then(Value::as_bool), Some(false)); + assert_eq!(s(&t, "database"), "cdc"); + } + + #[test] + fn ch_url_tls_scheme_defaults_secure_port() { + let t = ch_table("clickhouses://u:p@ch.cloud/db?compression=zstd").unwrap(); + assert_eq!(i(&t, "port"), 9440); + assert_eq!(t.get("secure").and_then(Value::as_bool), Some(true)); + assert_eq!(s(&t, "compression"), "zstd"); + } + + #[test] + fn ch_url_secure_param_overrides_scheme() { + let t = ch_table("clickhouse://ch:9440/db?secure=true").unwrap(); + assert_eq!(t.get("secure").and_then(Value::as_bool), Some(true)); + assert_eq!(i(&t, "port"), 9440); + } + + #[test] + fn ch_url_rejects_non_bool_secure() { + assert!(matches!( + ch_table("clickhouse://ch/db?secure=maybe").unwrap_err(), + DsnError::NotBool { .. } + )); + } + + #[test] + fn bad_port_is_rejected() { + assert!(matches!( + source_table("postgres://h:0/d").unwrap_err(), + DsnError::BadPort(_) + )); + assert!(matches!( + source_table("postgres://h:99999/d").unwrap_err(), + DsnError::BadPort(_) + )); + } + + #[test] + fn non_utf8_component_is_rejected() { + assert!(matches!( + source_table("postgres://h/d?slot=%FF").unwrap_err(), + DsnError::BadEncoding(_) + )); + } +} diff --git a/src/emit/ch_ddl.rs b/src/emit/ch_ddl.rs index dcb594f9..e3f19ff9 100644 --- a/src/emit/ch_ddl.rs +++ b/src/emit/ch_ddl.rs @@ -826,6 +826,41 @@ pub fn render_create_table( ))) } +/// CH `UNKNOWN_DATABASE` +const UNKNOWN_DATABASE: i32 = 81; + +/// Create `[ch] database` when the server doesn't have it, over a session on +/// `default` — every other client names it in the handshake, which CH refuses +/// outright for an absent database, so no connected client can create its own. +/// `Ok(false)` when it was already there. Sibling databases (per-namespace +/// `target_database`) go through the applicator's own `ensure_database` instead +pub async fn ensure_boot_database(cfg: &EmitterConfig) -> Result { + match connect_client(cfg).await { + Ok(_) => Ok(false), + Err(EmitterError::Client(e)) if e.server_code == UNKNOWN_DATABASE => { + let mut on_default = cfg.clone(); + on_default.database = "default".into(); + let mut client = connect_client(&on_default).await?; + exec_drain( + &mut client, + &format!( + "CREATE DATABASE IF NOT EXISTS {}", + quote_ident(&cfg.database) + ), + cfg.insert_timeout, + ) + .await?; + tracing::info!( + target: "walshadow::ch_ddl", + database = %cfg.database, + "created destination database", + ); + Ok(true) + } + Err(e) => Err(e), + } +} + /// `CREATE TABLE IF NOT EXISTS` rendered from an existing mapping — the /// re-create path for a mapped dest dropped under strategy=drop. Columns /// come from the mapping (the emitter's INSERT contract), not the diff --git a/src/lib.rs b/src/lib.rs index ead12eeb..cf3af1eb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ pub mod ch; pub mod column_rules; pub mod config; pub mod decode; +pub mod dsn; pub mod emit; pub mod filter; pub mod fs; @@ -55,7 +56,9 @@ pub use emit::{ch_ddl, ch_emitter, pipeline}; #[doc(hidden)] pub use filter::{catalog_tracker, classify, filter_segment, main_data, pg_class_decoder, rewrite}; #[doc(hidden)] -pub use ops::{bridge, control, metrics, oracle, preflight, retention, trace}; +pub use ops::{ + bridge, control, ctl, init, introspect, metrics, oracle, preflight, retention, trace, +}; #[doc(hidden)] pub use source::{ boundary_hold, catalog_capture, manifest, queueing_record_sink, segment_sink, shadow_stream, diff --git a/src/ops/control.rs b/src/ops/control.rs index 866c2259..2bf2ad80 100644 --- a/src/ops/control.rs +++ b/src/ops/control.rs @@ -11,12 +11,16 @@ use anyhow::{Context, Result, bail}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{UnixListener, UnixStream}; use tokio::sync::Mutex; -use tokio_postgres::{Client, NoTls}; +use tokio_postgres::Client; use toml::{Table, Value}; use walrus::pg::backup::format_pg_lsn; +use crate::config::SourceConn; +use crate::introspect; use crate::metrics::MetricsRegistry; +use crate::schema::RelName; +use crate::source_feed::open_sql_client; /// Holds the running session's resolver so the control socket + SIGHUP can /// trigger a live `reload()`. The daemon streams one session; there is no @@ -65,9 +69,10 @@ impl Reloader { #[derive(Clone)] pub struct SharedCtx { pub ch_config: PathBuf, - /// CLI-arg `[source]` defaults; the config file overrides them, matching the - /// daemon's connection resolution (see `ch_emitter::load_effective`). - pub source_base: Table, + /// CLI-arg `[source]` / `[ch]` defaults; the config file overrides them, + /// matching the daemon's connection resolution + /// (see `ch_emitter::load_effective`). + pub cli_base: Table, pub metrics: MetricsRegistry, pub reloader: Arc, /// Prevents concurrent fragment updates from overwriting each other @@ -256,44 +261,33 @@ fn frag_path(ch_config: &Path) -> PathBuf { } async fn get_config(ctx: &SharedCtx) -> Result
{ - Ok(crate::ch_emitter::load_effective(&ctx.ch_config, ctx.source_base.clone()).await?) + Ok(crate::ch_emitter::load_effective(&ctx.ch_config, ctx.cli_base.clone()).await?) } async fn tables_list<'a>(ctx: &SharedCtx, req: &Request<'a>) -> Result { let root = get_config(ctx).await?; let client = pg_connect(&root).await?; let ns = req.config.get("namespace").and_then(Value::as_str); - let base = "SELECT n.nspname, c.relname, c.relreplident \ - FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ - WHERE c.relkind = 'r' AND n.nspname NOT IN ('pg_catalog','information_schema') \ - AND n.nspname NOT LIKE 'pg\\_%'"; - let rows = if let Some(ns) = ns { - client - .query(&format!("{base} AND n.nspname=$1 ORDER BY 1,2"), &[&ns]) - .await - } else { - client.query(&format!("{base} ORDER BY 1,2"), &[]).await - } - .context("list tables")?; + let listed = introspect::tables(&client, ns) + .await + .context("list tables")?; let selected: ahash::HashSet<(String, String)> = selected_tables(&root).into_iter().collect(); - let mut arr = Vec::with_capacity(rows.len()); - for r in rows { - let ns: String = r.get(0); - let rel: String = r.get(1); - let ident: i8 = r.get(2); - let mut t = Table::new(); - t.insert( - "selected".into(), - selected.contains(&(ns.clone(), rel.clone())).into(), - ); - t.insert( - "replica_identity".into(), - Value::String((ident as u8 as char).to_string()), - ); - t.insert("namespace".into(), ns.into()); - t.insert("name".into(), rel.into()); - arr.push(Value::Table(t)); - } + let arr = listed + .into_iter() + .map(|t| { + let key = (t.rel.namespace.to_string(), t.rel.name.to_string()); + let mut row = Table::new(); + row.insert("selected".into(), selected.contains(&key).into()); + row.insert( + "replica_identity".into(), + Value::String(t.replica_identity.to_string()), + ); + row.insert("has_row_key".into(), t.has_row_key().into()); + row.insert("namespace".into(), key.0.into()); + row.insert("name".into(), key.1.into()); + Value::Table(row) + }) + .collect(); let mut out = Table::new(); out.insert("tables".into(), Value::Array(arr)); Ok(ok_toml(&out)) @@ -302,16 +296,12 @@ async fn tables_list<'a>(ctx: &SharedCtx, req: &Request<'a>) -> Result { async fn schemas_list(ctx: &SharedCtx) -> Result { let root = get_config(ctx).await?; let client = pg_connect(&root).await?; - let rows = client - .query( - "SELECT nspname FROM pg_namespace \ - WHERE nspname NOT IN ('pg_catalog','information_schema') \ - AND nspname NOT LIKE 'pg\\_%' ORDER BY 1", - &[], - ) + let names: Vec = introspect::schemas(&client) .await - .context("list schemas")?; - let names: Vec = rows.iter().map(|r| r.get::<_, String>(0).into()).collect(); + .context("list schemas")? + .into_iter() + .map(Value::String) + .collect(); let mut out = Table::new(); out.insert("schemas".into(), Value::Array(names)); Ok(ok_toml(&out)) @@ -326,25 +316,18 @@ async fn columns_list<'a>(ctx: &SharedCtx, req: &Request<'a>) -> Result }; let root = get_config(ctx).await?; let client = pg_connect(&root).await?; - let rows = client - .query( - "SELECT a.attname, format_type(a.atttypid, a.atttypmod), a.attnotnull \ - FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid \ - JOIN pg_namespace n ON n.oid=c.relnamespace \ - WHERE n.nspname=$1 AND c.relname=$2 AND a.attnum>0 AND NOT a.attisdropped \ - ORDER BY a.attnum", - &[&ns, &rel], - ) + let arr = introspect::columns(&client, &RelName::new(ns, rel)) .await - .context("list columns")?; - let mut arr = Vec::with_capacity(rows.len()); - for r in rows { - let mut t = Table::new(); - t.insert("name".into(), r.get::<_, String>(0).into()); - t.insert("type".into(), r.get::<_, String>(1).into()); - t.insert("notnull".into(), r.get::<_, bool>(2).into()); - arr.push(Value::Table(t)); - } + .context("list columns")? + .into_iter() + .map(|c| { + let mut t = Table::new(); + t.insert("name".into(), c.name.into()); + t.insert("type".into(), c.pg_type.into()); + t.insert("notnull".into(), c.notnull.into()); + Value::Table(t) + }) + .collect(); let mut out = Table::new(); out.insert("columns".into(), Value::Array(arr)); Ok(ok_toml(&out)) @@ -494,51 +477,18 @@ async fn save(path: &Path, root: &Table) -> Result<()> { Ok(()) } -fn render(v: &Value) -> String { - match v { - Value::String(s) => s.clone(), - other => other.to_string(), - } -} - -fn str_at(root: &Table, section: &str, key: &str) -> String { - root.get(section) - .and_then(Value::as_table) - .and_then(|t| t.get(key)) - .map(render) - .unwrap_or_default() -} - -// TODO: use daemon catalog, direct NoTls connection cannot inspect TLS-only sources +// TODO: use daemon catalog rather than a second connection per request async fn pg_connect(root: &Table) -> Result { - let host = str_at(root, "source", "host"); - if host.is_empty() { + let conn = SourceConn::from_table(root).map_err(|e| anyhow::anyhow!("[source] {e}"))?; + if conn.host.is_empty() { bail!("source host not set"); } - let mut cfg = tokio_postgres::Config::new(); - cfg.host(&host) - .port(str_at(root, "source", "port").parse().unwrap_or(5432)) - .dbname(nonempty(str_at(root, "source", "dbname"), "postgres")) - .user(nonempty(str_at(root, "source", "user"), "postgres")); - let pw = str_at(root, "source", "password"); - if !pw.is_empty() { - cfg.password(&pw); - } - let (client, conn) = cfg - .connect(NoTls) + open_sql_client(&conn.to_pg_config()) .await - .context("connect source postgres")?; - tokio::spawn(async move { - let _ = conn.await; - }); - Ok(client) + .with_context(|| format!("connect source {}", conn.endpoint())) } // ---- misc ----------------------------------------------------------------- - -fn nonempty(v: String, default: &str) -> String { - if v.is_empty() { default.to_string() } else { v } -} fn set_mode_600(path: &Path) -> Result<()> { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) @@ -549,6 +499,18 @@ fn set_mode_600(path: &Path) -> Result<()> { mod tests { use super::*; + /// Scalar at `[section] key`, for asserting fragment edits + fn str_at(root: &Table, section: &str, key: &str) -> String { + root.get(section) + .and_then(Value::as_table) + .and_then(|t| t.get(key)) + .map(|v| match v { + Value::String(s) => s.clone(), + other => other.to_string(), + }) + .unwrap_or_default() + } + fn cfg(toml: &str) -> Table { if toml.is_empty() { Table::new() @@ -598,7 +560,7 @@ mod tests { fn ctx_at(dir: &Path) -> SharedCtx { SharedCtx { ch_config: dir.join("ch-config.toml"), - source_base: Table::new(), + cli_base: Table::new(), metrics: MetricsRegistry::new(), reloader: Arc::new(Reloader::default()), frag_lock: Arc::new(Mutex::new(())), diff --git a/src/ops/ctl.rs b/src/ops/ctl.rs new file mode 100644 index 00000000..a7594877 --- /dev/null +++ b/src/ops/ctl.rs @@ -0,0 +1,427 @@ +//! `walshadow-stream ctl` word list → control request, and back +//! +//! The socket speaks TOML fragments ([`crate::control`]). This is the +//! translation layer above the verbs: `ctl add public users` becomes an +//! `apply` of `[table.public.users] replicate = true`, and the +//! `[[tables]]` reply comes back as aligned columns + +use std::path::PathBuf; + +use anyhow::Result; +use clap::{Parser, Subcommand, ValueEnum}; +use toml::{Table, Value}; + +#[derive(Debug)] +pub struct Command { + pub verb: String, + pub body: Table, + /// Raw verbs take their body from stdin; sugar builds its own + pub reads_stdin: bool, +} + +#[derive(Debug, Parser)] +#[command( + name = "walshadow-stream ctl", + about = "Control a running walshadow-stream daemon" +)] +pub struct Cli { + #[arg( + long, + env = "WALSHADOW_CONTROL_SOCKET", + default_value = "/run/walshadow/control.sock" + )] + socket: PathBuf, + #[command(subcommand)] + command: CtlCommand, +} + +impl Cli { + pub fn into_parts(self) -> Result<(PathBuf, Command)> { + Ok((self.socket, self.command.into_command()?)) + } +} + +#[derive(Debug, Subcommand)] +enum CtlCommand { + /// Show stream position, lag, and pause state + Status, + /// Show effective config with passwords masked + Show, + /// List source tables, optionally within one schema + Tables { schema: Option }, + /// List source schemas + Schemas, + /// List source columns + Columns { schema: String, table: String }, + /// Start replicating one table + Add { + schema: String, + table: String, + #[arg(long, value_enum)] + initial_load: Option, + }, + /// Stop replicating one table, retain ClickHouse table + Remove { schema: String, table: String }, + /// Freeze WAL consumption + Pause, + /// Resume WAL consumption + Resume, + /// Repoint source endpoint + Source { url: String }, + /// Repoint destination endpoint + #[command(alias = "destination")] + Dest { url: String }, + /// Re-read config, same as SIGHUP + Reload, + /// Apply TOML fragment from stdin + Apply, + /// Unset keys named by TOML fragment from stdin + Unset, + #[command(external_subcommand)] + External(Vec), +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +#[value(rename_all = "snake_case")] +enum InitialLoad { + None, + Copy, + BaseBackup, + ObjectStore, +} + +impl InitialLoad { + fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Copy => "copy", + Self::BaseBackup => "base_backup", + Self::ObjectStore => "object_store", + } + } +} + +impl CtlCommand { + fn into_command(self) -> Result { + match self { + Self::Status => Ok(sugar("status", Table::new())), + Self::Show => Ok(sugar("show", Table::new())), + Self::Reload => Ok(sugar("reload", Table::new())), + Self::Schemas => Ok(sugar("schemas", Table::new())), + Self::Pause => Ok(sugar( + "apply", + section("stream", pair("paused", true.into())), + )), + Self::Resume => Ok(sugar( + "apply", + section("stream", pair("paused", false.into())), + )), + Self::Tables { schema } => { + let mut body = Table::new(); + if let Some(schema) = schema { + body.insert("namespace".into(), schema.into()); + } + Ok(sugar("tables", body)) + } + Self::Columns { schema, table } => { + let mut body = pair("namespace", schema.into()); + body.insert("relname".into(), table.into()); + Ok(sugar("columns", body)) + } + Self::Add { + schema, + table, + initial_load, + } => { + let mut block = pair("replicate", true.into()); + if let Some(mode) = initial_load { + block.insert("initial_load".into(), mode.as_str().into()); + } + Ok(sugar("apply", table_block(&schema, &table, block))) + } + Self::Remove { schema, table } => { + let block = pair("replicate", false.into()); + Ok(sugar("apply", table_block(&schema, &table, block))) + } + Self::Source { url } => Ok(sugar( + "apply", + section("source", crate::dsn::source_table(&url)?), + )), + Self::Dest { url } => Ok(sugar("apply", section("ch", crate::dsn::ch_table(&url)?))), + Self::Apply => Ok(raw("apply")), + Self::Unset => Ok(raw("unset")), + Self::External(words) => { + let verb = words.first().expect("external subcommand has a name"); + Ok(raw(verb)) + } + } + } +} + +/// Words after `ctl`. Unknown verbs pass through with a stdin body so a +/// newer daemon's verbs work against an older CLI +pub fn parse>(words: &[S]) -> Result { + let args = std::iter::once("walshadow-stream ctl".to_owned()) + .chain(words.iter().map(|word| word.as_ref().to_owned())); + Cli::try_parse_from(args)?.command.into_command() +} + +/// Human view of a reply payload. Unknown shapes pass through verbatim +pub fn render(verb: &str, payload: &str) -> String { + let parsed: Option
= payload.parse().ok(); + let Some(root) = parsed else { + return payload.into(); + }; + match verb { + "tables" => render_tables(&root).unwrap_or_else(|| payload.into()), + "schemas" => root + .get("schemas") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(Value::as_str) + .collect::>() + .join("\n") + }) + .unwrap_or_else(|| payload.into()), + "columns" => render_columns(&root).unwrap_or_else(|| payload.into()), + _ => payload.into(), + } +} + +fn render_tables(root: &Table) -> Option { + let rows = root.get("tables")?.as_array()?; + let cells: Vec<(bool, String, String, String)> = rows + .iter() + .filter_map(Value::as_table) + .map(|t| { + ( + t.get("selected").and_then(Value::as_bool).unwrap_or(false), + str_of(t, "namespace"), + str_of(t, "name"), + match t.get("has_row_key").and_then(Value::as_bool) { + Some(false) => "no row key".into(), + _ => format!("identity {}", str_of(t, "replica_identity")), + }, + ) + }) + .collect(); + let ns_width = cells.iter().map(|c| c.1.len()).max().unwrap_or(0); + let name_width = cells.iter().map(|c| c.2.len()).max().unwrap_or(0); + Some( + cells + .iter() + .map(|(selected, ns, name, note)| { + let mark = if *selected { '*' } else { ' ' }; + format!("{mark} {ns:>() + .join("\n"), + ) +} + +fn render_columns(root: &Table) -> Option { + let rows = root.get("columns")?.as_array()?; + let cells: Vec<(String, String, bool)> = rows + .iter() + .filter_map(Value::as_table) + .map(|t| { + ( + str_of(t, "name"), + str_of(t, "type"), + t.get("notnull").and_then(Value::as_bool).unwrap_or(false), + ) + }) + .collect(); + let width = cells.iter().map(|c| c.0.len()).max().unwrap_or(0); + Some( + cells + .iter() + .map(|(name, ty, notnull)| { + let null = if *notnull { " not null" } else { "" }; + format!(" {name:>() + .join("\n"), + ) +} + +fn str_of(t: &Table, key: &str) -> String { + t.get(key).and_then(Value::as_str).unwrap_or("").into() +} + +fn sugar(verb: &str, body: Table) -> Command { + Command { + verb: verb.into(), + body, + reads_stdin: false, + } +} + +fn raw(verb: &str) -> Command { + Command { + verb: verb.into(), + body: Table::new(), + reads_stdin: true, + } +} + +fn pair(key: &str, value: Value) -> Table { + let mut t = Table::new(); + t.insert(key.into(), value); + t +} + +fn section(name: &str, body: Table) -> Table { + let mut root = Table::new(); + root.insert(name.into(), Value::Table(body)); + root +} + +/// `[table..]`, the two name parts kept separate all the way down +fn table_block(ns: &str, rel: &str, block: Table) -> Table { + section( + "table", + section(ns, { + let mut t = Table::new(); + t.insert(rel.into(), Value::Table(block)); + t + }), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; + + fn cmd(words: &[&str]) -> Command { + parse(words).unwrap() + } + fn err(words: &[&str]) -> String { + parse(words).unwrap_err().to_string() + } + + #[test] + fn add_builds_an_opt_in_fragment() { + let c = cmd(&["add", "public", "users", "--initial-load", "copy"]); + assert_eq!(c.verb, "apply"); + assert!(!c.reads_stdin); + assert_eq!( + toml::to_string(&c.body).unwrap(), + "[table.public.users]\ninitial_load = \"copy\"\nreplicate = true\n" + ); + } + + #[test] + fn add_defaults_to_no_initial_load_key() { + let c = cmd(&["add", "public", "users"]); + assert_eq!( + toml::to_string(&c.body).unwrap(), + "[table.public.users]\nreplicate = true\n" + ); + } + + #[test] + fn remove_opts_out_without_touching_other_tables() { + let c = cmd(&["remove", "public", "users"]); + assert_eq!( + toml::to_string(&c.body).unwrap(), + "[table.public.users]\nreplicate = false\n" + ); + } + + #[test] + fn dotted_name_is_not_split_into_a_pair() { + let error = err(&["add", "public.users"]); + assert!(error.contains("
"), "{error}"); + } + + #[test] + fn pause_and_resume_flip_one_flag() { + assert_eq!( + toml::to_string(&cmd(&["pause"]).body).unwrap(), + "[stream]\npaused = true\n" + ); + assert_eq!( + toml::to_string(&cmd(&["resume"]).body).unwrap(), + "[stream]\npaused = false\n" + ); + } + + #[test] + fn source_and_dest_take_urls() { + let c = cmd(&["source", "postgres://repl@db:5433/app"]); + let source = c.body["source"].as_table().unwrap(); + assert_eq!(source["host"].as_str(), Some("db")); + assert_eq!(source["port"].as_integer(), Some(5433)); + let c = cmd(&["dest", "clickhouses://ch.cloud/cdc"]); + let ch = c.body["ch"].as_table().unwrap(); + assert_eq!(ch["secure"].as_bool(), Some(true)); + assert_eq!(ch["port"].as_integer(), Some(9440)); + } + + #[test] + fn raw_verbs_still_read_stdin() { + assert!(cmd(&["apply"]).reads_stdin); + assert!(cmd(&["unset"]).reads_stdin); + assert!(cmd(&["some-future-verb", "--new-option", "value"]).reads_stdin); + assert!(!cmd(&["status"]).reads_stdin); + } + + #[test] + fn cli_parses_socket_with_typed_command() { + let cli = Cli::try_parse_from(["ctl", "--socket", "/tmp/custom.sock", "status"]).unwrap(); + let (socket, command) = cli.into_parts().unwrap(); + assert_eq!(socket, PathBuf::from("/tmp/custom.sock")); + assert_eq!(command.verb, "status"); + } + + #[test] + fn bad_initial_load_mode_is_rejected() { + let error = err(&["add", "public", "users", "--initial-load", "warp"]); + assert!(error.contains("invalid value 'warp'"), "{error}"); + assert!(error.contains("base_backup"), "{error}"); + } + + #[test] + fn initial_load_accepts_equals_syntax() { + let c = cmd(&["add", "public", "users", "--initial-load=object_store"]); + assert_eq!( + toml::to_string(&c.body).unwrap(), + "[table.public.users]\ninitial_load = \"object_store\"\nreplicate = true\n" + ); + } + + #[test] + fn known_verbs_reject_extra_arguments() { + let error = err(&["status", "extra"]); + assert!(error.contains("unexpected argument 'extra'"), "{error}"); + } + + #[test] + fn generated_help_lists_control_verbs() { + let help = Cli::command().render_long_help().to_string(); + assert!(help.contains("Commands:"), "{help}"); + assert!(help.contains("add"), "{help}"); + assert!(help.contains("Start replicating one table"), "{help}"); + } + + #[test] + fn tables_render_marks_selection_and_missing_keys() { + let payload = "[[tables]]\nselected = true\nnamespace = \"public\"\nname = \"users\"\n\ + replica_identity = \"d\"\nhas_row_key = true\n\ + [[tables]]\nselected = false\nnamespace = \"public\"\nname = \"audit\"\n\ + replica_identity = \"d\"\nhas_row_key = false\n"; + assert_eq!( + render("tables", payload), + "* public users identity d\n public audit no row key" + ); + } + + #[test] + fn unknown_payload_passes_through() { + assert_eq!(render("status", "paused = false\n"), "paused = false\n"); + assert_eq!(render("tables", "not toml ["), "not toml ["); + } +} diff --git a/src/ops/init.rs b/src/ops/init.rs new file mode 100644 index 00000000..6e855364 --- /dev/null +++ b/src/ops/init.rs @@ -0,0 +1,570 @@ +//! First-run setup: probe both ends, pick tables, write the config +//! +//! `init` takes two connection URLs, validates both endpoints, reports +//! what the source must fix before replication can start (with the SQL +//! that fixes it), then writes `[source]` / `[ch]` / `[table.*]`, so a +//! first run needs no hand-authored TOML +//! +//! Tables land as opt-in intents (`replicate = true`, no `columns`), so +//! shapes come from the source descriptor and CH tables auto-create — +//! see [`crate::opt_in::apply_table_opt_in`], which boot and reload share + +use std::collections::BTreeSet; +use std::io::{IsTerminal, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use toml::{Table, Value}; + +use crate::ch_emitter::EmitterConfig; +use crate::config::SourceConn; +use crate::introspect::{self, SourceTable}; +use crate::preflight::{self, MIN_SERVER_VERSION_NUM, PreflightError, SourceInputs}; +use crate::schema::RelName; +use crate::source_feed::open_sql_client; + +pub const SOURCE_URL_ENV: &str = "WALSHADOW_SOURCE_URL"; +pub const CH_URL_ENV: &str = "WALSHADOW_CH_URL"; + +pub struct InitOpts { + /// Config to write. Refuses to clobber unless `force` + pub config: PathBuf, + pub source_url: Option, + pub ch_url: Option, + /// Explicit `(namespace, name)` selection; empty defers to `all_tables` + /// or the picker + pub tables: Vec, + pub all_tables: bool, + /// Restrict listing and `--all-tables` to one schema + pub namespace: Option, + /// `none` / `copy` / `base_backup` / `object_store` + pub initial_load: String, + pub force: bool, +} + +/// Probe, select, write. Returns after the config exists on disk; starting +/// the daemon stays the caller's step +pub async fn run(opts: InitOpts) -> Result<()> { + if opts.config.exists() && !opts.force { + bail!( + "{} exists — pass --force to overwrite, or edit it in place", + opts.config.display() + ); + } + let source_url = resolve_url( + opts.source_url.clone(), + SOURCE_URL_ENV, + "Source Postgres URL", + "postgres://user:password@host:5432/dbname", + )?; + let source = crate::dsn::source_table(&source_url)?; + let conn = SourceConn::from_table(&wrap("source", source.clone())) + .map_err(|e| anyhow::anyhow!("[source] {e}"))?; + + println!("\nsource {}", conn.endpoint()); + let client = open_sql_client(&conn.to_pg_config()) + .await + .with_context(|| format!("connect source {}", conn.endpoint()))?; + let version_num = server_version_num(&client).await?; + println!(" ✓ reachable, PostgreSQL {}", version_text(version_num)); + let findings = probe_source(&client, version_num, conn.slot.as_deref()).await?; + for line in &findings { + println!("{line}"); + } + + let ch_url = resolve_url( + opts.ch_url.clone(), + CH_URL_ENV, + "ClickHouse URL", + "clickhouse://user:password@host:9000/database", + )?; + let ch = crate::dsn::ch_table(&ch_url)?; + let ch_cfg = EmitterConfig::from_table(&wrap("ch", ch.clone())) + .map_err(|e| anyhow::anyhow!("[ch] {e}"))?; + println!("\ndestination {}:{}", ch_cfg.host, ch_cfg.port); + let created_db = crate::ch_ddl::ensure_boot_database(&ch_cfg) + .await + .context("connect ClickHouse")?; + let ch_client = crate::ch::connect_client(&ch_cfg) + .await + .context("connect ClickHouse")?; + match ch_client.server_info() { + Some(i) => println!( + " ✓ reachable, ClickHouse {}.{}.{}", + i.version_major, i.version_minor, i.version_patch + ), + None => println!(" ✓ reachable"), + } + if created_db { + println!(" ✓ created database {}", ch_cfg.database); + } + + let listed = introspect::tables(&client, opts.namespace.as_deref()) + .await + .context("list source tables")?; + let picked = select_tables(&listed, &opts)?; + if picked.is_empty() { + println!( + "\nno tables selected — add them later with `walshadow-stream ctl add
`" + ); + } + + let doc = build_config(source, ch, &picked, &opts.initial_load); + write_config(&opts.config, &doc)?; + + println!("\nwrote {}", opts.config.display()); + for rel in &picked { + println!(" {} {}", rel.namespace, rel.name); + } + let blocking = findings.iter().any(|f| f.starts_with(" ✗")); + if blocking { + println!("\nfix the ✗ items above first — the daemon refuses to start until then"); + } + println!( + "\nstart streaming:\n walshadow-stream --ch-config {} …", + opts.config.display() + ); + Ok(()) +} + +/// Everything `init` can check with an ordinary SQL connection, rendered +/// as `✓` / `✗` lines with the SQL that clears each `✗` +async fn probe_source( + client: &tokio_postgres::Client, + version_num: i32, + slot: Option<&str>, +) -> Result> { + let mut out = Vec::new(); + let report = preflight::source(SourceInputs { + source_version_num: version_num, + source_sql: client, + slot, + ch_config: None, + }) + .await + .context("pre-flight")?; + if report.is_ok() { + out.push(format!( + " ✓ wal_level = logical, server_version_num ≥ {MIN_SERVER_VERSION_NUM}" + )); + } + for e in &report.errors { + out.push(format!(" ✗ {e}")); + if let Some(fix) = remedy(e) { + out.push(format!(" {fix}")); + } + } + + let row = client + .query_one( + "SELECT rolsuper OR rolreplication, current_user::text FROM pg_roles \ + WHERE rolname = current_user", + &[], + ) + .await + .context("read replication privilege")?; + let can_replicate: bool = row.get(0); + let role: String = row.get(1); + if can_replicate { + out.push(format!(" ✓ {role} may start replication")); + } else { + out.push(format!(" ✗ {role} lacks REPLICATION")); + out.push(format!(" ALTER ROLE {role} REPLICATION;")); + } + + let senders: i32 = client + .query_one("SELECT current_setting('max_wal_senders')::int", &[]) + .await + .context("read max_wal_senders")? + .get(0); + if senders < 1 { + out.push(" ✗ max_wal_senders = 0, no walsender slot for the shadow".into()); + out.push(" ALTER SYSTEM SET max_wal_senders = 8; -- restart required".into()); + } + + if let Some(mismatch) = shadow_major_mismatch(version_num) { + out.push(format!(" ✗ {mismatch}")); + } + Ok(out) +} + +/// Source SQL that clears a pre-flight finding, where one exists +fn remedy(e: &PreflightError) -> Option { + match e { + PreflightError::WalLevel { .. } => Some( + "ALTER SYSTEM SET wal_level = logical; -- restart required (managed PG: set it in the provider console)" + .into(), + ), + PreflightError::SlotMissing { slot } => { + Some(format!("SELECT pg_create_physical_replication_slot('{slot}');")) + } + PreflightError::BadReplicaIdentity { rel, .. } => Some(format!( + "ALTER TABLE {}.{} REPLICA IDENTITY FULL; -- or add a PRIMARY KEY", + rel.namespace, rel.name + )), + _ => None, + } +} + +/// Shadow is a physical clone, so its postmaster must be the source's +/// major. The binaries come off `PATH`, so check the ones this host has +fn shadow_major_mismatch(source_version_num: i32) -> Option { + let source_major = source_version_num / 10_000; + let out = std::process::Command::new("initdb") + .arg("--version") + .output(); + let Ok(out) = out else { + return Some(format!( + "no `initdb` on PATH — the shadow needs PostgreSQL {source_major} binaries \ + (image: rebuild with --build-arg PG_MAJOR={source_major})" + )); + }; + let text = String::from_utf8_lossy(&out.stdout); + let local = text + .split_whitespace() + .last() + .and_then(|v| v.split(['.', 'd', 'r', 'b']).next()) + .and_then(|v| v.parse::().ok()); + match local { + Some(major) if major == source_major => None, + Some(major) => Some(format!( + "shadow binaries are PostgreSQL {major}, source is {source_major}; \ + a basebackup-cloned shadow cannot span majors \ + (image: rebuild with --build-arg PG_MAJOR={source_major})" + )), + None => Some(format!( + "could not read `initdb --version` ({})", + text.trim() + )), + } +} + +/// Explicit list, then `--all-tables`, then the interactive picker +fn select_tables(listed: &[SourceTable], opts: &InitOpts) -> Result> { + if !opts.tables.is_empty() { + let known: ahash::HashSet<&RelName> = listed.iter().map(|t| &t.rel).collect(); + for rel in &opts.tables { + if !known.contains(rel) { + bail!("{}.{} not found on source", rel.namespace, rel.name); + } + } + return Ok(opts.tables.clone()); + } + let replicable: Vec<&SourceTable> = listed.iter().filter(|t| t.has_row_key()).collect(); + if opts.all_tables { + for t in listed.iter().filter(|t| !t.has_row_key()) { + println!( + " skipping {}.{} — {}", + t.rel.namespace, + t.rel.name, + t.row_key_note() + ); + } + return Ok(replicable.iter().map(|t| t.rel.clone()).collect()); + } + if !std::io::stdin().is_terminal() { + bail!("no terminal for the table picker — pass --all-tables or --table
"); + } + print_table_menu(listed); + // Re-prompt rather than discard the probing work already done + loop { + let answer = prompt("\nTables (numbers, ranges, `all`, or empty for none): ")?; + match resolve_picked(listed, &answer) { + Ok(picked) => return Ok(picked), + Err(e) => println!(" {e}"), + } + } +} + +/// One picker answer against the listing: every index must name a table +/// walshadow can replicate +fn resolve_picked(listed: &[SourceTable], answer: &str) -> Result> { + let mut out = Vec::new(); + for idx in parse_selection(answer, listed.len())? { + let t = &listed[idx]; + if !t.has_row_key() { + bail!( + "{} {} has no row key: {}", + t.rel.namespace, + t.rel.name, + t.row_key_note() + ); + } + out.push(t.rel.clone()); + } + Ok(out) +} + +fn print_table_menu(listed: &[SourceTable]) { + println!("\nsource tables:"); + let ns_width = listed + .iter() + .map(|t| t.rel.namespace.len()) + .max() + .unwrap_or(0); + let name_width = listed.iter().map(|t| t.rel.name.len()).max().unwrap_or(0); + for (i, t) in listed.iter().enumerate() { + // `!` marks what the picker will refuse, so the reason reads as a fix + let mark = if t.has_row_key() { ' ' } else { '!' }; + println!( + " {:>3}. {mark} {: Result> { + let answer = answer.trim(); + if answer.is_empty() || answer.eq_ignore_ascii_case("none") { + return Ok(Vec::new()); + } + if answer.eq_ignore_ascii_case("all") { + return Ok((0..len).collect()); + } + let mut out = BTreeSet::new(); + for part in answer.split(',').map(str::trim).filter(|s| !s.is_empty()) { + let (lo, hi) = match part.split_once('-') { + Some((a, b)) => (index(a, len)?, index(b, len)?), + None => { + let i = index(part, len)?; + (i, i) + } + }; + if lo > hi { + bail!("range {part:?} runs backwards"); + } + out.extend(lo..=hi); + } + Ok(out.into_iter().collect()) +} + +fn index(raw: &str, len: usize) -> Result { + let n: usize = raw + .trim() + .parse() + .with_context(|| format!("{raw:?} is not a number"))?; + if n < 1 || n > len { + bail!("{n} out of range 1..={len}"); + } + Ok(n - 1) +} + +fn build_config(source: Table, ch: Table, picked: &[RelName], initial_load: &str) -> Table { + let mut root = Table::new(); + root.insert("source".into(), Value::Table(source)); + root.insert("ch".into(), Value::Table(ch)); + if picked.is_empty() { + return root; + } + let mut tables = Table::new(); + for rel in picked { + let mut block = Table::new(); + block.insert("replicate".into(), true.into()); + block.insert("initial_load".into(), initial_load.into()); + let ns = tables + .entry(rel.namespace.to_string()) + .or_insert_with(|| Value::Table(Table::new())); + if let Value::Table(ns) = ns { + ns.insert(rel.name.to_string(), Value::Table(block)); + } + } + root.insert("table".into(), Value::Table(tables)); + root +} + +/// 0600: the file carries both passwords +fn write_config(path: &Path, doc: &Table) -> Result<()> { + if let Some(dir) = path.parent() + && !dir.as_os_str().is_empty() + { + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + } + let body = format!( + "# Written by `walshadow-stream init`. Edit freely, or drive it live\n\ + # over the control socket (`walshadow-stream ctl …`), which writes\n\ + # its own fragments beside this file and never rewrites it.\n\n{}", + toml::to_string(doc).context("serialize config")? + ); + std::fs::write(path, body).with_context(|| format!("write {}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .with_context(|| format!("chmod 0600 {}", path.display()))?; + } + Ok(()) +} + +fn wrap(section: &str, body: Table) -> Table { + let mut root = Table::new(); + root.insert(section.into(), Value::Table(body)); + root +} + +/// Flag, then env, then prompt. Env keeps credentials out of `ps` and +/// shell history +fn resolve_url(flag: Option, env: &str, label: &str, example: &str) -> Result { + if let Some(url) = flag.filter(|s| !s.trim().is_empty()) { + return Ok(url); + } + if let Ok(url) = std::env::var(env) + && !url.trim().is_empty() + { + return Ok(url); + } + if !std::io::stdin().is_terminal() { + bail!("no {label}: pass the flag or set {env} (eg {example})"); + } + let answer = prompt(&format!("{label} [{example}]: "))?; + if answer.trim().is_empty() { + bail!("no {label} given"); + } + Ok(answer.trim().into()) +} + +fn prompt(text: &str) -> Result { + print!("{text}"); + std::io::stdout().flush()?; + let mut line = String::new(); + std::io::stdin().read_line(&mut line)?; + Ok(line) +} + +async fn server_version_num(client: &tokio_postgres::Client) -> Result { + let row = client + .query_one("SELECT current_setting('server_version_num')::int", &[]) + .await + .context("read server_version_num")?; + Ok(row.get(0)) +} + +fn version_text(version_num: i32) -> String { + format!("{}.{}", version_num / 10_000, version_num % 10_000) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn listed() -> Vec { + vec![ + SourceTable { + rel: RelName::new("public", "users"), + replica_identity: 'd', + has_pk: true, + }, + SourceTable { + rel: RelName::new("public", "audit"), + replica_identity: 'd', + has_pk: false, + }, + ] + } + + fn opts() -> InitOpts { + InitOpts { + config: "/tmp/walshadow-init-test.toml".into(), + source_url: None, + ch_url: None, + tables: Vec::new(), + all_tables: false, + namespace: None, + initial_load: "copy".into(), + force: false, + } + } + + #[test] + fn all_tables_skips_keyless_relations() { + let picked = select_tables( + &listed(), + &InitOpts { + all_tables: true, + ..opts() + }, + ) + .unwrap(); + assert_eq!(picked, vec![RelName::new("public", "users")]); + } + + #[test] + fn explicit_table_must_exist_on_source() { + let e = select_tables( + &listed(), + &InitOpts { + tables: vec![RelName::new("public", "ghost")], + ..opts() + }, + ) + .unwrap_err(); + assert!(e.to_string().contains("not found on source"), "{e}"); + } + + #[test] + fn selection_accepts_numbers_ranges_and_all() { + assert_eq!(parse_selection("all", 3).unwrap(), vec![0, 1, 2]); + assert_eq!(parse_selection("1,3", 3).unwrap(), vec![0, 2]); + assert_eq!(parse_selection(" 2-3 ", 3).unwrap(), vec![1, 2]); + assert_eq!(parse_selection("", 3).unwrap(), Vec::::new()); + assert_eq!(parse_selection("2,2", 3).unwrap(), vec![1]); + } + + #[test] + fn picker_refuses_a_keyless_table_by_name() { + let e = resolve_picked(&listed(), "1,2").unwrap_err().to_string(); + assert!(e.contains("public audit has no row key"), "{e}"); + assert_eq!( + resolve_picked(&listed(), "1").unwrap(), + vec![RelName::new("public", "users")] + ); + } + + #[test] + fn selection_rejects_out_of_range_and_backwards() { + assert!(parse_selection("4", 3).is_err()); + assert!(parse_selection("0", 3).is_err()); + assert!(parse_selection("3-1", 3).is_err()); + assert!(parse_selection("x", 3).is_err()); + } + + #[test] + fn config_holds_opt_in_blocks_keyed_by_pair() { + let doc = build_config( + crate::dsn::source_table("postgres://u@h/d").unwrap(), + crate::dsn::ch_table("clickhouse://h/db").unwrap(), + &[RelName::new("public", "users")], + "copy", + ); + let rendered = toml::to_string(&doc).unwrap(); + let parsed = EmitterConfig::from_toml_str(&rendered).unwrap(); + assert!(parsed.tables.is_empty(), "opt-in carries no pinned columns"); + let row = parsed + .table_opt_ins + .get(&RelName::new("public", "users")) + .expect("opt-in intent"); + assert_eq!(row.replicate, Some(true)); + assert_eq!(row.initial_load.as_deref(), Some("copy")); + } + + #[test] + fn written_config_round_trips_connection_settings() { + let doc = build_config( + crate::dsn::source_table("postgres://repl:pw@src:5433/app?sslmode=require").unwrap(), + crate::dsn::ch_table("clickhouses://u:p@ch/cdc").unwrap(), + &[], + "none", + ); + let conn = SourceConn::from_table(&doc).unwrap(); + assert_eq!(conn.host, "src"); + assert_eq!(conn.port, 5433); + assert_eq!(conn.dbname, "app"); + let ch = EmitterConfig::from_table(&doc).unwrap(); + assert_eq!(ch.host, "ch"); + assert_eq!(ch.port, 9440); + assert!(ch.secure); + } +} diff --git a/src/ops/introspect.rs b/src/ops/introspect.rs new file mode 100644 index 00000000..112d1b7a --- /dev/null +++ b/src/ops/introspect.rs @@ -0,0 +1,149 @@ +//! Source-PG catalog reads shared by the control socket and `init` +//! +//! Plain SQL over an ordinary (non-replication) connection: what a +//! table picker needs to show, and what pre-flight needs to judge a +//! relation replicable + +use tokio_postgres::Client; + +use crate::schema::RelName; + +/// `pg_class` row as the picker sees it +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceTable { + pub rel: RelName, + /// `pg_class.relreplident`: `d` default, `n` nothing, `f` full, `i` index + pub replica_identity: char, + pub has_pk: bool, +} + +impl SourceTable { + /// DELETE needs a row key, and CH needs an `ORDER BY` to collapse on + pub fn has_row_key(&self) -> bool { + match self.replica_identity { + 'd' => self.has_pk, + 'n' => false, + _ => true, + } + } + + /// Why the picker refuses it, or how it identifies rows + pub fn row_key_note(&self) -> &'static str { + match self.replica_identity { + 'd' if self.has_pk => "primary key", + 'd' => "no primary key — add one, or SET REPLICA IDENTITY FULL", + 'n' => "REPLICA IDENTITY NOTHING — deletes cannot be replicated", + 'f' => "REPLICA IDENTITY FULL", + 'i' => "REPLICA IDENTITY USING INDEX", + _ => "unknown replica identity", + } + } +} + +/// User relations, ordered by (namespace, name). `namespace` filters to one +/// schema; `None` lists every non-system schema +pub async fn tables( + client: &Client, + namespace: Option<&str>, +) -> Result, tokio_postgres::Error> { + const BASE: &str = "SELECT n.nspname, c.relname, c.relreplident::text, \ + EXISTS (SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary) \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind = 'r' AND n.nspname NOT IN ('pg_catalog','information_schema') \ + AND n.nspname NOT LIKE 'pg\\_%'"; + let rows = match namespace { + Some(ns) => { + client + .query(&format!("{BASE} AND n.nspname = $1 ORDER BY 1,2"), &[&ns]) + .await? + } + None => client.query(&format!("{BASE} ORDER BY 1,2"), &[]).await?, + }; + Ok(rows + .iter() + .map(|r| { + let ident: String = r.get(2); + SourceTable { + rel: RelName::new(r.get(0), r.get(1)), + replica_identity: ident.chars().next().unwrap_or('?'), + has_pk: r.get(3), + } + }) + .collect()) +} + +/// Non-system schema names +pub async fn schemas(client: &Client) -> Result, tokio_postgres::Error> { + let rows = client + .query( + "SELECT nspname FROM pg_namespace \ + WHERE nspname NOT IN ('pg_catalog','information_schema') \ + AND nspname NOT LIKE 'pg\\_%' ORDER BY 1", + &[], + ) + .await?; + Ok(rows.iter().map(|r| r.get(0)).collect()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceColumn { + pub name: String, + pub pg_type: String, + pub notnull: bool, +} + +/// Live columns of one relation in `attnum` order +pub async fn columns( + client: &Client, + rel: &RelName, +) -> Result, tokio_postgres::Error> { + let (ns, name): (&str, &str) = (&rel.namespace, &rel.name); + let rows = client + .query( + "SELECT a.attname, format_type(a.atttypid, a.atttypmod), a.attnotnull \ + FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid \ + JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE n.nspname = $1 AND c.relname = $2 AND a.attnum > 0 \ + AND NOT a.attisdropped ORDER BY a.attnum", + &[&ns, &name], + ) + .await?; + Ok(rows + .iter() + .map(|r| SourceColumn { + name: r.get(0), + pg_type: r.get(1), + notnull: r.get(2), + }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn table(ident: char, has_pk: bool) -> SourceTable { + SourceTable { + rel: RelName::new("public", "t"), + replica_identity: ident, + has_pk, + } + } + + #[test] + fn default_identity_needs_a_primary_key() { + assert!(table('d', true).has_row_key()); + assert!(!table('d', false).has_row_key()); + } + + #[test] + fn full_and_index_identity_carry_their_own_key() { + assert!(table('f', false).has_row_key()); + assert!(table('i', false).has_row_key()); + } + + #[test] + fn nothing_identity_never_has_a_key() { + assert!(!table('n', true).has_row_key()); + } +} diff --git a/src/ops/mod.rs b/src/ops/mod.rs index 6b1ad1f8..ae38acce 100644 --- a/src/ops/mod.rs +++ b/src/ops/mod.rs @@ -1,5 +1,8 @@ pub mod bridge; pub mod control; +pub mod ctl; +pub mod init; +pub mod introspect; pub mod metrics; pub mod oracle; pub mod preflight; diff --git a/src/source/source_feed.rs b/src/source/source_feed.rs index 11cf13e1..339878d2 100644 --- a/src/source/source_feed.rs +++ b/src/source/source_feed.rs @@ -529,7 +529,7 @@ impl SourceFeed { /// Mirrors wal-rus's transport choice: unix socket when `host` starts /// with `/`, TLS-or-plain TCP otherwise. Shared with the COPY backfiller /// ([`crate::backfill::copy_backfill`]), which opens its own session per backfill. -pub(crate) async fn open_sql_client(cfg: &PgConfig) -> Result { +pub async fn open_sql_client(cfg: &PgConfig) -> Result { let mut tp_cfg = tokio_postgres::Config::new(); tp_cfg .user(cfg.user.as_str()) diff --git a/tests/init_e2e.rs b/tests/init_e2e.rs new file mode 100644 index 00000000..804d6770 --- /dev/null +++ b/tests/init_e2e.rs @@ -0,0 +1,152 @@ +//! `walshadow-stream init` against a live source PG + ClickHouse. +//! +//! Covers two connection URLs through bootable config output. Live run proves: +//! +//! - a unix-socket `postgres://…?host=%2F…` URL reaches a socket-only +//! cluster, and the written `[source]` round-trips back to it +//! - the pre-flight report reads the real `wal_level` / role privileges +//! - a table without a row key is skipped, one with a PK is opted in +//! - `[ch] database` is created when absent — CH refuses the handshake +//! for a missing one, so the daemon could not create its own +//! +//! Skipped silently without `initdb` or the `clickhouse` multitool. + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::fs; +use std::time::Duration; + +use walshadow::ch_emitter::EmitterConfig; +use walshadow::config::SourceConn; +use walshadow::init::{InitOpts, run}; +use walshadow::schema::RelName; +use walshadow::shadow::{Shadow, ShadowConfig}; + +fn make_source(tmp: &tempfile::TempDir) -> Shadow { + let mut cfg = ShadowConfig::new( + tmp.path().join("source-data"), + tmp.path().join("source-filtered"), + ); + cfg.port = fx::PG_SOURCE_PORT; + cfg.socket_dir = tmp.path().join("source-sock"); + cfg.ctl_timeout = Duration::from_secs(60); + fs::create_dir_all(&cfg.filter_out_dir).unwrap(); + fs::create_dir_all(&cfg.socket_dir).unwrap(); + Shadow::new(cfg) +} + +/// Percent-encode a socket dir into the libpq URL spelling +fn socket_url(socket_dir: &std::path::Path, dbname: &str) -> String { + let encoded = socket_dir.to_str().unwrap().replace('/', "%2F"); + format!( + "postgres://postgres@/{dbname}?host={encoded}&port={}&sslmode=disable", + fx::PG_SOURCE_PORT + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn init_probes_both_ends_and_writes_a_bootable_config() { + if !fx::pg_available() { + eprintln!("skip: no initdb on PATH"); + return; + } + if !fx::clickhouse_available() { + eprintln!("skip: no clickhouse binary on PATH"); + return; + } + + let slot = fx::Ports::alloc(); + let tmp = tempfile::tempdir().unwrap(); + + let source = make_source(&tmp); + source.initdb().expect("initdb source"); + source.write_base_conf().expect("source base conf"); + fx::append_source_conf(&source).expect("append source conf"); + source.start().expect("start source"); + let _src_stop = fx::StopOnDrop { sh: &source }; + + // One replicable relation, one the picker must refuse + source + .apply_schema_dump( + "CREATE SCHEMA app;\n\ + CREATE TABLE app.users (id int4 PRIMARY KEY, name text);\n\ + CREATE TABLE app.audit (msg text);\n", + ) + .expect("source schema"); + + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, slot.ch_tcp, slot.ch_http).expect("spawn ch"); + + let config = tmp.path().join("ch-config.toml"); + run(InitOpts { + config: config.clone(), + source_url: Some(socket_url(&source.config().socket_dir, "postgres")), + // `cdc` does not exist yet: init has to create it + ch_url: Some(format!( + "clickhouse://default@127.0.0.1:{}/cdc", + slot.ch_tcp + )), + tables: Vec::new(), + all_tables: true, + namespace: None, + initial_load: "copy".into(), + force: false, + }) + .await + .expect("init"); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(&config).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "config carries passwords"); + } + + let written = fs::read_to_string(&config).expect("read config"); + let root: toml::Table = written.parse().expect("config parses"); + + let conn = SourceConn::from_table(&root).expect("[source]"); + assert_eq!(conn.host, source.config().socket_dir.to_str().unwrap()); + assert_eq!(conn.port, fx::PG_SOURCE_PORT); + assert_eq!(conn.dbname, "postgres"); + + let cfg = EmitterConfig::from_table(&root).expect("[ch]"); + assert_eq!(cfg.port, slot.ch_tcp); + assert_eq!(cfg.database, "cdc"); + + let users = cfg + .table_opt_ins + .get(&RelName::new("app", "users")) + .expect("keyed table opted in"); + assert_eq!(users.replicate, Some(true)); + assert_eq!(users.initial_load.as_deref(), Some("copy")); + assert!( + !cfg.table_opt_ins + .contains_key(&RelName::new("app", "audit")), + "keyless table stays out of the config", + ); + + let exists = ch.query("EXISTS DATABASE cdc").expect("query ch"); + assert_eq!(exists.trim(), "1", "init created the destination database"); + + // Second run refuses to clobber, and says how to override + let again = run(InitOpts { + config: config.clone(), + source_url: Some(socket_url(&source.config().socket_dir, "postgres")), + ch_url: Some(format!( + "clickhouse://default@127.0.0.1:{}/cdc", + slot.ch_tcp + )), + tables: Vec::new(), + all_tables: true, + namespace: None, + initial_load: "copy".into(), + force: false, + }) + .await + .expect_err("existing config is not overwritten"); + assert!(again.to_string().contains("--force"), "{again}"); +} diff --git a/tests/schema_evolution_pinned.rs b/tests/schema_evolution_pinned.rs index 3c058d9e..a59eaca6 100644 --- a/tests/schema_evolution_pinned.rs +++ b/tests/schema_evolution_pinned.rs @@ -1,9 +1,8 @@ //! Pinned-table DDL baseline — `ALTER ADD COLUMN` on an operator-pinned //! relation propagates to ClickHouse without any priming DML. //! -//! The docker-demo scenario (`docker/DEMO.md`): `demo.users` is pinned in -//! `ch-config`, gets no traffic, then the presenter runs `ALTER TABLE -//! demo.users ADD COLUMN signup_ts …`. Pre-fix, the very first descriptor +//! `demo.users` is pinned in `ch-config` and gets no traffic before `ALTER TABLE +//! demo.users ADD COLUMN signup_ts …`. Pre-fix, first descriptor //! fetch already carried the post-ALTER shape, `prev_known` was cold for //! the oid → `Added` → `apply_added` skips the pinned dest → CH never grew //! the column. The startup `seed_baseline` warms `prev_known` with the