diff --git a/README.md b/README.md index c22b9487..a757f44b 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,8 @@ refusal — never a silently wrong or incomplete result: with the desired definition: it plans as one online `VALIDATE CONSTRAINT`, and refuses with `validated-constraint-drift` if the constraint changed after planning. - **Invalid-index recovery has no CLI verb yet** — a failed concurrent index - build's leftover is reported with a typed state, and the proven removal + build's leftover is reported with a typed state (and named by `diff` as a + typed plan advisory), and the proven removal (`executor.RebuildAbandonedIndex`, or `executor.DropAbandonedIndex` when the caller must not rebuild) is library-only; from the CLI the [runbook](docs/invalid-index-recovery.md) applies. diff --git a/demo/README.md b/demo/README.md index edcf46b3..dc51e153 100644 --- a/demo/README.md +++ b/demo/README.md @@ -13,8 +13,8 @@ The tour walks five sections, each runnable on its own via | Section | What it shows | Writes? | | --------- | ------------------------------------------------------------------------------------------------------------------- | ------------------- | | `dryrun` | One statement per planner route, reason, and disposition: metadata-only, fast-default, binary-coercible, the safer idioms, a rewrite-required suggestion, type-rewrite, volatile-default, an app-breaking rename (refused as rewrite-required), a destructive drop (planned under `--allow-destructive`, refused as `destructive-change` without it), relocation, and a refusal | no | -| `diff` | The declarative front door: a routed convergence plan for an existing table, for a missing one, and a typed create-shape refusal (exit code 2) for a missing table declared `IF NOT EXISTS` | no | -| `pull` | Existing-database onboarding: export one desired file per demo table, then prove each produces a zero-change `diff` | no | +| `diff` | The declarative front door: a routed convergence plan for an existing table, for a missing one, the plan advisories for a table carrying an invalid index (the rebuild the desired file names, and the leftover it does not), and a typed create-shape refusal (exit code 2) for a missing table declared `IF NOT EXISTS` | no | +| `pull` | Existing-database onboarding: export one desired file per demo table, then prove each produces a zero-change `diff` — except the table seeded with an invalid index, whose baseline re-diffs to its one rebuild | no | | `offline` | `lint` (gates on error findings), `suggest` (advises), `fmt` (canonicalizes) — no database | no | | `exec` | Real executions: a native add, the concurrent index substitution, the four-step `SET NOT NULL` sequence, a genuine type rewrite run through copy-and-swap (`executed-via-copy-and-swap`, exit code 0), a structured refusal (exit code 2) for a rewrite the copy-and-swap engine does not carry yet, and the `--accept-blocking` passthrough: the plain `DROP INDEX` refused, refused again with a mismatched acknowledgement (exit code 1, nothing runs), then committed without online safety (exit code 3) along with a `REINDEX TABLE`, and a `DROP COLUMN` refused as `destructive-change` (exit code 2) until `--allow-destructive` names the table | yes (seeded tables) | @@ -38,7 +38,9 @@ collapses the `SET NOT NULL` sequence turns the job red), statement counts — and on exit codes (`0` success, `2` refusal, `3` accepted-blocking commit, `1` lint gate and mismatched acknowledgement). The `pull` step has no JSON mode, so check mode asserts only its exit code and exported -file count, then uses each `diff --json` report to assert zero statements. It never +file count, then uses each `diff --json` report to assert zero statements (one +`create-index` plus its `invalid-index-rebuild` advisory for the table seeded +with an invalid index). It never asserts on human-facing prose, which is free to change. CI runs this as the `demo` job ("smoke test (built pg-sprite artifact)"): the built `bin/pg-sprite` exercised end-to-end against compose PostgreSQL. diff --git a/demo/desired/contacts_no_index.sql b/demo/desired/contacts_no_index.sql new file mode 100644 index 00000000..702d06ef --- /dev/null +++ b/demo/desired/contacts_no_index.sql @@ -0,0 +1,4 @@ +CREATE TABLE contacts ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + email text NOT NULL +); diff --git a/demo/desired/contacts_rebuild.sql b/demo/desired/contacts_rebuild.sql new file mode 100644 index 00000000..2b13e3ef --- /dev/null +++ b/demo/desired/contacts_rebuild.sql @@ -0,0 +1,6 @@ +CREATE TABLE contacts ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + email text NOT NULL +); + +CREATE UNIQUE INDEX contacts_email_key ON contacts (email); diff --git a/demo/seed.sql b/demo/seed.sql index 1340fff8..4577306e 100644 --- a/demo/seed.sql +++ b/demo/seed.sql @@ -2,7 +2,7 @@ -- recreates everything, so every `make demo` starts from the same state. -- Identity (not serial) keys so the declarative diff never meets a -- sequence-backed default it must refuse. -DROP TABLE IF EXISTS users, orders, widgets CASCADE; +DROP TABLE IF EXISTS users, orders, widgets, contacts CASCADE; CREATE TABLE users ( id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, @@ -33,6 +33,24 @@ FROM generate_series(1, 5000) g; -- desired-state file describes, so the diff tour's plans do not see it. CREATE INDEX orders_total_idx ON orders (total); +-- A table carrying the leftover of a failed concurrent index build: the +-- unique build below runs over duplicate emails, so it fails after creating +-- its catalog entry and leaves contacts_email_key behind as an invalid +-- index. The diff tour shows the plan's advisory for it, and the pull tour +-- shows the freshly exported baseline re-diffing to that one rebuild. +-- CONCURRENTLY cannot run inside a transaction, so the failure is tolerated +-- here rather than caught in a DO block. +CREATE TABLE contacts ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + email text NOT NULL +); + +INSERT INTO contacts (email) VALUES ('dup@example.com'), ('dup@example.com'); + +\set ON_ERROR_STOP 0 +CREATE UNIQUE INDEX CONCURRENTLY contacts_email_key ON contacts (email); +\set ON_ERROR_STOP 1 + -- Dedicated fixture for atomic desired-state row security. CREATE SCHEMA IF NOT EXISTS demo_security; DROP TABLE IF EXISTS demo_security.demo_rls; diff --git a/demo/tour.sh b/demo/tour.sh index ffea6e8a..8e5cfcb8 100755 --- a/demo/tour.sh +++ b/demo/tour.sh @@ -78,7 +78,7 @@ dry_run() { else assert_eq "dry-run refusal exit of [$sql]" 2 "$status" fi - assert_eq "plan format_version of [$sql]" 14 "$(jq -r '.format_version' <<<"$out")" + assert_eq "plan format_version of [$sql]" 15 "$(jq -r '.format_version' <<<"$out")" assert_eq "disposition of [$sql]" "$disposition" "$(jq -r '.disposition' <<<"$out")" assert_eq "route of [$sql]" "$route" "$(jq -r '.statements[0].route' <<<"$out")" assert_eq "reason of [$sql]" "$reason" "$(jq -r '.statements[0].decisions[0].reason' <<<"$out")" @@ -254,7 +254,7 @@ diff_plan() { if [ "$CHECK" = 1 ]; then out=$("$PGS" diff --url "$PG_DSN" --desired "$desired" --schema public --json) || status=$? assert_eq "diff exit of [$desired]" 0 "$status" - assert_eq "diff format_version of [$desired]" 14 "$(jq -r '.format_version' <<<"$out")" + assert_eq "diff format_version of [$desired]" 15 "$(jq -r '.format_version' <<<"$out")" assert_eq "statement count of [$desired]" "$count" "$(jq -r '.statements | length' <<<"$out")" case "$(jq -r '.statements[0].sql' <<<"$out")" in *"$fragment"*) ;; @@ -265,6 +265,33 @@ diff_plan() { fi } +# diff_advisory desired_file statement_count advisory_kind +# +# A desired file for the seeded table that carries an invalid index (the +# leftover of the failed unique build in seed.sql). The diff never drops an +# unfinished build, so the plan carries an advisory naming it instead +# (plan.Report.Advisories, schemadiff.AdvisoryKinds()): the rebuild kind when +# the desired file names the index, the undeclared kind when it does not — +# in which case the plan has zero statements but is not silent about the +# leftover. The advisory sits outside the statements and the fingerprint, so +# the exit code stays 0. +diff_advisory() { + local desired="$1" count="$2" kind="$3" out status=0 + step "diff --desired $desired (expect advisory)" + if [ "$CHECK" = 1 ]; then + out=$("$PGS" diff --url "$PG_DSN" --desired "$desired" --schema public --json) || status=$? + assert_eq "diff exit of [$desired]" 0 "$status" + assert_eq "diff format_version of [$desired]" 15 "$(jq -r '.format_version' <<<"$out")" + assert_eq "diff disposition of [$desired]" execute "$(jq -r '.disposition' <<<"$out")" + assert_eq "statement count of [$desired]" "$count" "$(jq -r '.statements | length' <<<"$out")" + assert_eq "advisory count of [$desired]" 1 "$(jq -r '.advisories | length' <<<"$out")" + assert_eq "advisory kind of [$desired]" "$kind" "$(jq -r '.advisories[0].kind' <<<"$out")" + assert_eq "advisory index of [$desired]" contacts_email_key "$(jq -r '.advisories[0].index' <<<"$out")" + else + "$PGS" diff --url "$PG_DSN" --desired "$desired" --schema public || echo "(exit $?)" + fi +} + # diff_refused desired_file cause # # A greenfield desired file whose create shape the create path refuses: the @@ -282,7 +309,7 @@ diff_refused() { if [ "$CHECK" = 1 ]; then out=$("$PGS" diff --url "$PG_DSN" --desired "$desired" --schema public --json) || status=$? assert_eq "diff refusal exit of [$desired]" 2 "$status" - assert_eq "diff format_version of [$desired]" 14 "$(jq -r '.format_version' <<<"$out")" + assert_eq "diff format_version of [$desired]" 15 "$(jq -r '.format_version' <<<"$out")" assert_eq "diff disposition of [$desired]" refuse "$(jq -r '.disposition' <<<"$out")" assert_eq "diff reason of [$desired]" unsupported-statement "$(jq -r '.statements[0].reason' <<<"$out")" assert_eq "diff class of [$desired]" "$class" "$(jq -r '.statements[0].class' <<<"$out")" @@ -304,12 +331,18 @@ run_pull() { "$PGS" pull --url "$PG_DSN" --schema public --out "$out_dir" >/dev/null || status=$? assert_eq "pull exit" 0 "$status" file_count=$(find "$out_dir" -maxdepth 1 -type f -name '*.sql' | wc -l | tr -d ' ') - # One file per table in demo/seed.sql (orders, users). - assert_eq "pulled file count" 2 "$file_count" + # One file per table in demo/seed.sql (contacts, orders, users). + assert_eq "pulled file count" 3 "$file_count" else "$PGS" pull --url "$PG_DSN" --schema public --out "$out_dir" || echo "(exit $?)" fi + # Every exported file re-diffs to zero statements — except the one table + # carrying an invalid index (contacts, seeded that way): pull renders the + # index like any other, so its baseline re-diffs to exactly one + # create-index, and the plan's rebuild advisory names the entry. That + # lone create is the signal that a just-exported table carries an + # unfinished build (docs/pull.md). for desired in "$out_dir"/*.sql; do step "zero-diff proof: $desired" status=0 @@ -317,9 +350,19 @@ run_pull() { out=$("$PGS" diff --url "$PG_DSN" --schema public --desired "$desired" --json) || status=$? assert_eq "diff exit of [$desired]" 0 "$status" # Plan report contract version (plan.FormatVersion), same pin as diff_plan. - assert_eq "diff format_version of [$desired]" 14 "$(jq -r '.format_version' <<<"$out")" + assert_eq "diff format_version of [$desired]" 15 "$(jq -r '.format_version' <<<"$out")" assert_eq "diff disposition of [$desired]" execute "$(jq -r '.disposition' <<<"$out")" - assert_eq "zero diff of [$desired]" 0 "$(jq -r '.statements | length' <<<"$out")" + case "$(basename "$desired")" in + contacts.sql) + assert_eq "rebuild diff of [$desired]" 1 "$(jq -r '.statements | length' <<<"$out")" + assert_eq "rebuild kind of [$desired]" create-index "$(jq -r '.statements[0].kind' <<<"$out")" + assert_eq "advisory kind of [$desired]" invalid-index-rebuild "$(jq -r '.advisories[0].kind' <<<"$out")" + ;; + *) + assert_eq "zero diff of [$desired]" 0 "$(jq -r '.statements | length' <<<"$out")" + assert_eq "no advisories of [$desired]" 0 "$(jq -r '.advisories | length' <<<"$out")" + ;; + esac else "$PGS" diff --url "$PG_DSN" --schema public --desired "$desired" || echo "(exit $?)" fi @@ -370,6 +413,8 @@ run_diff() { heading "Declarative front door: routed convergence plans (read-only)" diff_plan desired/users_v2.sql 2 "ADD COLUMN bio" diff_plan desired/widgets_v1.sql 2 "CREATE TABLE" + diff_advisory desired/contacts_rebuild.sql 1 invalid-index-rebuild + diff_advisory desired/contacts_no_index.sql 0 invalid-index-undeclared diff_refused desired/gadgets_if_not_exists.sql if-not-exists } diff --git a/docs/capabilities.md b/docs/capabilities.md index 68328175..2ab4a9e2 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -264,7 +264,7 @@ or the only online form is a direct statement the operator runs). | Table shape | Status | Engine path | Online-safety problem? | Behavior and why | | --- | --- | --- | --- | --- | -| Plain tables + their indexes | ✅ | native, as-is | Yes | `diff`, `pull`, and desired-file rendering round-trip the canonical model. The model carries each index's validity (`pg_index.indisvalid`): on a plain table an invalid entry is a concurrent build that did not finish — abandoned, or still running — so a live entry with the desired name and definition does not deliver the desired index, and `diff` plans it as a `create-index` change. `diff` never emits a drop for an invalid entry, whatever the desired file says about its name: a plain `DROP INDEX` blocks the table and cannot tell abandoned debris from a build still in progress. A `pull`ed baseline of such a table therefore re-diffs to that one rebuild rather than to zero. On a partitioned parent an invalid index means unattached partition indexes, not an unfinished build, so validity plays no part in the parent's comparison | +| Plain tables + their indexes | ✅ | native, as-is | Yes | `diff`, `pull`, and desired-file rendering round-trip the canonical model. The model carries each index's validity (`pg_index.indisvalid`): on a plain table an invalid entry is a concurrent build that did not finish — abandoned, or still running — so a live entry with the desired name and definition does not deliver the desired index, and `diff` plans it as a `create-index` change. `diff` never emits a drop for an invalid entry, whatever the desired file says about its name: a plain `DROP INDEX` blocks the table and cannot tell abandoned debris from a build still in progress. Every invalid entry the diff observes and leaves in place is named on the plan report as an advisory (`invalid-index-rebuild` when the desired file names it, `invalid-index-undeclared` when it does not), outside the statements and the fingerprint, so an empty plan over such a table reads as 'no changes to run' rather than converged. A `pull`ed baseline of such a table therefore re-diffs to that one rebuild rather than to zero. On a partitioned parent an invalid index means unattached partition indexes, not an unfinished build, so validity plays no part in the parent's comparison | | A live check constraint or foreign key left `NOT VALID` | ✅ | native, as-is | Yes | The model carries each constraint's validity (`pg_constraint.convalidated`). A live constraint left `NOT VALID` with the desired definition, which an interrupted online add leaves behind, plans as one `VALIDATE CONSTRAINT` statement (`validate-constraint`), which scans under `SHARE UPDATE EXCLUSIVE` while writes keep flowing, instead of a destructive drop and re-add. The statement names only the constraint, so execution reads the constraint again under that same lock and refuses with the permanent `validated-constraint-drift` code, validating nothing, if it was dropped or replaced under the same name since planning. A foreign key plans as `validate-foreign-key`, refused with `declarative-foreign-key` like every declarative foreign-key change. `CREATE TABLE` ignores `NOT VALID`, so a desired file cannot declare it: for a check constraint, `pull` renders the validated definition, and the baseline re-diffs to that one validation rather than to zero. `pull` refuses a table that owns a foreign key, validated or not, with `ErrUnrenderableForeignKey` | | Classic table inheritance (`INHERITS`) | 🟡 | native, planned flow | Yes | Typed refusal for both parents and children: the model cannot express inheritance edges, and flattening inherited columns would produce a silently lossy baseline. A desired file that declares `INHERITS` is refused before planning builds anything, since the scratch build would lock the live parent and add an inheritance edge to it | | Tables that own **or are referenced by** foreign keys | 🟡 | native, planned flow | Yes | Planned but not executed. A desired file may declare a foreign key to an unqualified table that exists in the target schema; `diff` shows a matching foreign key as no change and an added or dropped one as an `add-foreign-key` or `drop-foreign-key` statement refused with `declarative-foreign-key`, and `migrate --desired` refuses the whole plan. Planning binds each reference to an empty scratch copy of the live table, which needs `SELECT` on it, so planning takes only `ACCESS SHARE` on the live referenced table, the lock a plain `SELECT` takes, and never blocks its writes. Export still refuses both sides — an incoming FK cannot be expressed in the table's own desired file. Declarative FK execution (composite keys as the primary case, two-phase `NOT VALID` → `VALIDATE` execution) is planned | diff --git a/docs/cli-output-examples.md b/docs/cli-output-examples.md index 606073e5..63367aba 100644 --- a/docs/cli-output-examples.md +++ b/docs/cli-output-examples.md @@ -160,7 +160,7 @@ itself. ```console $ pg-sprite migrate --alter 'ALTER TABLE users ADD COLUMN note text' --dry-run --json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "public", "table": "users", @@ -201,7 +201,7 @@ plans the safer online sequence instead: the decision carries it in ```console $ pg-sprite migrate --alter 'ALTER TABLE users ADD CONSTRAINT users_email_key UNIQUE (email)' --dry-run --json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "public", "table": "users", @@ -303,7 +303,7 @@ column first, then build the constraint as a separate, named ```console $ pg-sprite migrate --alter 'ALTER TABLE users ADD COLUMN nickname text UNIQUE' --dry-run --json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "public", "table": "users", @@ -345,7 +345,7 @@ no native SQL to show. ```console $ pg-sprite migrate --alter 'ALTER TABLE users ALTER COLUMN id TYPE text' --dry-run --json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "public", "table": "users", @@ -405,7 +405,7 @@ column, a relocation — so the plan is `unavailable` and nothing executes. ```console $ pg-sprite migrate --alter 'ALTER TABLE users ADD COLUMN joined timestamptz DEFAULT clock_timestamp()' --dry-run --json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "public", "table": "users", @@ -442,7 +442,7 @@ The refusal cause is the report-level `reason`. ```console $ pg-sprite migrate --alter 'CREATE INDEX events_created_idx ON events (created)' --dry-run --json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "public", "table": "events", @@ -507,7 +507,7 @@ table. A mismatched acknowledgement executes nothing and exits 1. ```console $ pg-sprite migrate --alter 'ALTER TABLE users DROP COLUMN email' --dry-run --allow-destructive public.users --json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "public", "table": "users", @@ -634,7 +634,7 @@ CREATE TABLE users ( ```console $ pg-sprite diff --desired /tmp/users.sql --json { - "format_version": 14, + "format_version": 15, "source": "diff", "schema": "public", "table": "users", diff --git a/docs/invariants.md b/docs/invariants.md index b6fa2dcc..bbd02175 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -787,7 +787,8 @@ the proof's lock, every drop, and the requested build share **one** overall budg recovery over k quarantined entries costs at most the budget, not (k+1) budgets. The declarative diff leans on this proof without performing it: when desired names an index whose live entry is an unfinished build, `diff` plans the `create-index` alone and never a -drop, knowing the create cannot run as-is against the occupied name. The plan completes only +drop, knowing the create cannot run as-is against the occupied name, and says so with an +`invalid-index-rebuild` advisory on the plan report. The plan completes only through this invariant's proven removal — `RebuildAbandonedIndex`, or `DropAbandonedIndex` when the name must be cleared without completing the plan — or through an operator following the runbook; a plain `CREATE INDEX` on the occupied name fails as a duplicate @@ -1481,15 +1482,19 @@ a plain table is an unfinished concurrent build — abandoned, or still running never emits a `DROP INDEX` for it, whatever the desired file says about its name: a drop by name cannot tell the two apart, and dropping under a running build is the cleanup this invariant forbids. That keeps the cleanup half. It gives up the other half when the desired -file no longer names the index at all: the observation is discarded and the plan is empty, -so a desired state that removes an index the build never delivered reads as already -converged. The trade is preferred to the alternatives — a name-based drop, or a fabricated -change — because the plan stays executable and never destroys a build it cannot see; the -leftover surfaces at the next `pull` (see [pull.md](pull.md)) and clears per the -[invalid-index runbook](invalid-index-recovery.md). OC-1 reaches the same conclusion from -the other side: the diff cannot confirm the entry is abandoned, so it must not round the -uncertainty into a destructive change. *Enforced:* `pkg/schemadiff` never-drops-invalid -tests against real debris on a real database. +file no longer names the index at all: no statement is planned, so a desired state that +removes an index the build never delivered converges to zero statements. The observation is +not discarded, though — the plan report carries it as an `invalid-index-undeclared` +advisory (see [plan-report.md](plan-report.md#advisory-kinds-advisorieskind-diff-source-only)), outside the statements and +the fingerprint, so the empty plan says "no changes to run; the table carries an unfinished +build" rather than "converged". The trade is preferred to the alternatives — a name-based +drop, or a fabricated change — because the plan stays executable and never destroys a build +it cannot see; the leftover also surfaces at the next `pull` (see [pull.md](pull.md)) and +clears per the [invalid-index runbook](invalid-index-recovery.md). OC-1 reaches the same +conclusion from the other side: the diff cannot confirm the entry is abandoned, so it must +not round the uncertainty into a destructive change. *Enforced:* `pkg/schemadiff` +never-drops-invalid tests against real debris on a real database; `pkg/diffplan` proves the +empty plan carries the advisory and the unchanged fingerprint. ### OC-3 — Control requests are durable operator intent diff --git a/docs/limitations.md b/docs/limitations.md index 4fb40385..d40345cc 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -158,8 +158,8 @@ composition of the model boundaries above with those gates. At a glance: | Add a column | Converges. Runs as a bounded attempt of the submitted form, so the table-size guard applies (below). | | Widen a column type (`varchar(50)` → `varchar(255)`) | Converges — the same bounded attempt, under the same size guard. | | Add an index | Converges via `CREATE INDEX CONCURRENTLY`. Not size-guarded: long online work on a large table is the pattern's purpose. | -| Rebuild an index left invalid by an unfinished concurrent build | Plans as the same `create-index` change — the invalid entry, though it carries the desired name and definition, does not deliver the index, and the diff never emits a drop for it. Execution then meets the occupied name and stops by proof: a `failed` outcome carrying `invalid-index-abandoned` (the invalid-index row above); remove the leftover per the [runbook](invalid-index-recovery.md), or, as a library caller, run the create through `executor.RebuildAbandonedIndex`, and rerun. A build still in progress looks the same to the diff (`indisvalid` stays false until the build's final wait ends) and plans the same create; execution then stops with `invalid-index-build-in-flight` rather than build twice. | -| Remove or redefine an index that is currently invalid | The diff leaves the invalid entry alone: it never emits a drop for an entry that may be someone's build still running, so `diff` shows no change for a removed index and only the `create-index` for a redefined one (execution then meets the occupied name as in the row above). This also means the `_ccnew` leftover of an interrupted `REINDEX ... CONCURRENTLY`, which no desired file names, does not block the table's other changes; clear it per the [runbook](invalid-index-recovery.md). On a partitioned parent, where an invalid index means unattached partition indexes rather than an unfinished build, indexes are compared by name and definition alone. | +| Rebuild an index left invalid by an unfinished concurrent build | Plans as the same `create-index` change — the invalid entry, though it carries the desired name and definition, does not deliver the index, and the diff never emits a drop for it. The plan report says so alongside the statement: an `invalid-index-rebuild` [advisory](plan-report.md#advisory-kinds-advisorieskind-diff-source-only) names the entry, so the create is not mistaken for a plain new index. Execution then meets the occupied name and stops by proof: a `failed` outcome carrying `invalid-index-abandoned` (the invalid-index row above); remove the leftover per the [runbook](invalid-index-recovery.md), or, as a library caller, run the create through `executor.RebuildAbandonedIndex`, and rerun. A build still in progress looks the same to the diff (`indisvalid` stays false until the build's final wait ends) and plans the same create and advisory; execution then stops with `invalid-index-build-in-flight` rather than build twice. | +| Remove or redefine an index that is currently invalid | The diff leaves the invalid entry alone: it never emits a drop for an entry that may be someone's build still running, so `diff` plans no statement for a removed index and only the `create-index` for a redefined one (execution then meets the occupied name as in the row above). The plan is not silent about it: an [advisory](plan-report.md#advisory-kinds-advisorieskind-diff-source-only) names the entry the diff left in place — `invalid-index-undeclared` when the desired file no longer names the index, `invalid-index-rebuild` when it redefines it under the same name — so an empty plan over such a table reads as "no changes to run", not as converged. This also means the `_ccnew` leftover of an interrupted `REINDEX ... CONCURRENTLY`, which no desired file names, does not block the table's other changes; clear it per the [runbook](invalid-index-recovery.md). On a partitioned parent, where an invalid index means unattached partition indexes rather than an unfinished build, indexes are compared by name and definition alone and no advisory is raised. | | Add a constraint (`UNIQUE`, `CHECK`); `SET NOT NULL` | Converges via the safer online sequence; not size-guarded either. | | Relax a `NOT NULL` | Refused, whole plan: dropping `NOT NULL` discards the same guarantee its constraint form would, so it is destructive. Run it deliberately through the imperative front door — it executes natively there once `--allow-destructive` names the table — then rerun. | | Change an index or constraint definition | Refused, whole plan — the drop-and-recreate row above. | diff --git a/docs/plan-report.md b/docs/plan-report.md index 217d573d..8d493575 100644 --- a/docs/plan-report.md +++ b/docs/plan-report.md @@ -21,11 +21,12 @@ both shapes.) Every report carries `format_version`. A consumer that does not recognize the version must **reject the report** — never guess at field semantics. The version covers more than the field shape: the closed vocabularies below (sources, routes, backends, dispositions, -kinds, guidance, causes, classes, owners) and the fingerprint serialization are all pinned to it. Adding a vocabulary value or +kinds, guidance, causes, classes, owners, advisory kinds) and the fingerprint serialization are all pinned to it. Adding a vocabulary value or changing the fingerprint definition is a contract change and bumps `format_version`, even if no field is added or renamed. -The current version is **14**: version 14 added the `alter-column-type-in-place` guidance code (carried by a `destructive-change` refusal verdict, never by a report statement); version 13 added the `coordinated-constraint-rename` guidance code and classifies the rename of an index that backs a primary-key, unique, or exclusion constraint as `app-breaking-rename`; version 12 added the `expand-contract-column` and `coordinated-table-rename` guidance codes and made `app-breaking-rename` decisions yield the `rewrite-required` disposition; version 11 added the `cascading-drop` reason; version 10 added the `dependent-view` reason and the `dependent_views` decision field; version 9 added the `validate-constraint` and `validate-foreign-key` kinds; version 8 added the `replica-identity` reason; version 7 added the `set-generated-expression` kind; version 6 added the `add-foreign-key` and `drop-foreign-key` +The current version is **15**: version 15 added the report-level `advisories` field and its +closed kind vocabulary — the invalid indexes a `diff` observed and declined to drop; version 14 added the `alter-column-type-in-place` guidance code (carried by a `destructive-change` refusal verdict, never by a report statement); version 13 added the `coordinated-constraint-rename` guidance code and classifies the rename of an index that backs a primary-key, unique, or exclusion constraint as `app-breaking-rename`; version 12 added the `expand-contract-column` and `coordinated-table-rename` guidance codes and made `app-breaking-rename` decisions yield the `rewrite-required` disposition; version 11 added the `cascading-drop` reason; version 10 added the `dependent-view` reason and the `dependent_views` decision field; version 9 added the `validate-constraint` and `validate-foreign-key` kinds; version 8 added the `replica-identity` reason; version 7 added the `set-generated-expression` kind; version 6 added the `add-foreign-key` and `drop-foreign-key` kinds, the `foreign-key` cause, and the `declarative-foreign-key` reason; version 5 added `blocking_passthrough_eligible` to every refused statement; version 4 added the `class` and `owner` fields on the report and on refused statements, each drawn from a closed vocabulary (see Classes and Owners); @@ -66,6 +67,14 @@ consumer rendering either into a shared surface must clamp and escape them. | `owner` | string | `no-online-safety-problem` refusals only | Who owns the aggregate work pg-sprite has no online-safety reason to run (see Owners). Present exactly when `class` is `no-online-safety-problem`. | | `fingerprint` | string | always | The plan's stable identity (see Fingerprint). | | `statements` | array | always | The ordered plan; `[]` (never `null`) means nothing to do. | +| `advisories` | array | `diff` source, existing table, when non-empty | Typed observations the diff made about the live table that produced no statement (see Advisory fields): the invalid indexes it saw and declined to drop. Advisory only — the statements and the `destructive` gate are unaffected, and nothing in it executes. Read it alongside `statements`: an empty plan with an advisory present means there is nothing to run **and** the table is not clean. Absent when there is nothing to report. Also absent for a desired file that declares row security: that review path collects no advisories, so there an absent field does not mean the table has no invalid index. Explanatory: excluded from the fingerprint. | + +## Advisory fields + +| Field | Type | Presence | Meaning | +|---|---|---|---| +| `kind` | string | always | What the diff observed and what completes the plan (see Advisory kinds). Automation branches on this, never on prose. | +| `index` | string | always | The live index the observation is about. A catalog identifier — clamp and escape it before rendering into a shared surface. | ## Statement fields @@ -272,6 +281,19 @@ bumps `format_version`. Present exactly when `class` is `no-online-safety-proble | `validate-foreign-key` | Validates a live foreign key left `NOT VALID` whose definition otherwise matches the desired one. Always refused for execution (`declarative-foreign-key`). | | `create-index` | Creates an index. | +### Advisory kinds (`advisories[].kind`, diff source only) + +Both kinds describe an invalid index on a plain table — a concurrent build that did not +finish, abandoned or still running — which the diff never drops: a `DROP INDEX` by name +cannot tell the two apart ([OC-2, LK-5](invariants.md)). A partitioned parent's invalid +index means unattached partition indexes, not an unfinished build, and is never advised. A +consumer that meets a kind it does not recognize must surface it, never discard it. + +| Value | Meaning | +|---|---| +| `invalid-index-rebuild` | The desired schema names the index (same or redefined definition), so the plan carries its `create-index`. That create cannot run as-is against the occupied name; the plan completes only through the proven-removal path — `executor.RebuildAbandonedIndex` or the [invalid-index runbook](invalid-index-recovery.md) — which drops the entry after proving it abandoned. | +| `invalid-index-undeclared` | The desired schema does not name the index. The diff plans no drop for it, so no statement touches it: an empty plan is not evidence the table is clean. Clear an abandoned entry per the [runbook](invalid-index-recovery.md), or wait for a build still running to finish. | + ## Fingerprint `fingerprint` is the plan's stable identity: `sha256:` plus the hex digest over what would @@ -285,8 +307,10 @@ The serialization is exact and pinned by test: for each statement in plan order, canonical `sql`, `route`, `backend`, and `disposition`, then each `exec_sql` entry — every field followed by a unit separator (`0x1F`) — and close each statement with a record separator (`0x1E`). Explanatory fields (`decisions`, `kind`, `destructive`, `reason`, `cause`, -`guidance`) are excluded: a reworded reason does not change identity, but a rerouted, -resequenced, or rewritten plan does. An empty plan has a defined identity (the digest of no input). +`guidance`, and the report-level `advisories`) are excluded: a reworded reason does not +change identity, but a rerouted, resequenced, or rewritten plan does. An empty plan has a +defined identity (the digest of no input) — with or without advisories, since nothing in +an advisory executes. For a table that does not exist yet, `exec_sql` is the plain canonical build that the greenfield create path runs. It never substitutes `CONCURRENTLY` for an index on a table @@ -316,7 +340,7 @@ in `pkg/plan` — if the code drifts from this page, CI fails. ```json { - "format_version": 14, + "format_version": 15, "source": "alter", "schema": "app", "table": "orders", @@ -359,7 +383,7 @@ A desired state that drops an index and adds a column with a constant default: ```json { - "format_version": 14, + "format_version": 15, "source": "diff", "schema": "app", "table": "orders", @@ -426,7 +450,7 @@ A desired state for a table that does not exist yet, whose `CREATE TABLE` carrie ```json { - "format_version": 14, + "format_version": 15, "source": "diff", "schema": "app", "table": "gadgets", diff --git a/docs/pull.md b/docs/pull.md index a1dba30b..61d4e44a 100644 --- a/docs/pull.md +++ b/docs/pull.md @@ -87,13 +87,15 @@ The one table that does not round-trip to zero is one carrying an invalid index — a concurrent build that did not finish, or is still running. `pull` renders the index like any other (validity is not part of a desired file), and `diff` then plans exactly one `create-index` change for it: the entry carries the name -and definition but does not deliver the index, so the plan is its rebuild. A -freshly pulled baseline that re-diffs to a lone `create-index` is therefore the -signal that the table carries an unfinished build — nothing else makes a just- -exported file diff to a create. Query `pg_index.indisvalid` for the table's -indexes to confirm which entry it is, then clear an abandoned entry per the -[invalid-index runbook](invalid-index-recovery.md) — or wait for a build still -running to finish — before recording the baseline as verified. +and definition but does not deliver the index, so the plan is its rebuild. The +plan report names the entry in an `invalid-index-rebuild` advisory +(`advisories[]` in `diff --json`, a `warning[invalid-index-rebuild]` line in the +text output; see [plan-report.md](plan-report.md#advisory-kinds-advisorieskind-diff-source-only)), so a freshly +pulled baseline that re-diffs to a lone `create-index` announces itself as the +table carrying an unfinished build rather than leaving the reader to infer it. +Clear an abandoned entry per the [invalid-index runbook](invalid-index-recovery.md) +— or wait for a build still running to finish — before recording the baseline as +verified. ## Refused table shapes diff --git a/docs/schemabot-integration.md b/docs/schemabot-integration.md index 272acb01..1a18b3d4 100644 --- a/docs/schemabot-integration.md +++ b/docs/schemabot-integration.md @@ -599,8 +599,10 @@ and PostgreSQL `lock_timeout`). Contract points for the PostgreSQL adapter: `ACCESS EXCLUSIVE` and blocks reads too, so the policy's risk framing does not transfer. - **Recommended call sequence: dry-run, then apply.** The typed manual path for a rewrite-required refusal (`guidance`), the create path's typed shape refusal - (`statements[].cause`, `format_version` 3 and later), the plan `fingerprint`, and `table_exists` all - live on the [plan report](plan-report.md) — the dry run's output. The run verdict carries + (`statements[].cause`, `format_version` 3 and later), the invalid-index advisories the diff + declines to act on (`advisories[]`, `format_version` 15 and later; not collected for a desired + file that declares row security, so absence there proves nothing), the plan `fingerprint`, and + `table_exists` all live on the [plan report](plan-report.md) — the dry run's output. The run verdict carries the refusal reason as a typed field but its `detail` is display-only prose: an orchestrator that wants the typed guidance or cause must obtain the plan report first (dry-run for the imperative front door, `diff --json` for the declarative one), then apply, and treat the diff --git a/internal/cli/diff.go b/internal/cli/diff.go index 5fcab61d..0599b2e5 100644 --- a/internal/cli/diff.go +++ b/internal/cli/diff.go @@ -13,6 +13,7 @@ import ( "github.com/block/pg-sprite/pkg/plan" "github.com/block/pg-sprite/pkg/planner" "github.com/block/pg-sprite/pkg/router" + "github.com/block/pg-sprite/pkg/schemadiff" "github.com/block/pg-sprite/pkg/statement" "github.com/block/pg-sprite/pkg/verdict" ) @@ -103,13 +104,22 @@ func writeJSON(out io.Writer, report plan.Report) error { // The header points at migrate as the executing front door: running this // script directly bypasses the gate that refuses blocking statements. A // greenfield statement the create path refuses by shape is annotated with -// the plan's typed cause — the executor's own explanation. +// the plan's typed cause — the executor's own explanation. The plan's +// advisories — invalid indexes the diff declined to drop — are comment +// lines too, so a script over a table carrying debris says so and an empty +// plan does not claim the table converged. func writePlanText(out io.Writer, report plan.Report) error { if len(report.Statements) == 0 { - if _, err := fmt.Fprintln(out, "-- no changes: live table matches the desired schema"); err != nil { + if len(report.Advisories) == 0 { + if _, err := fmt.Fprintln(out, "-- no changes: live table matches the desired schema"); err != nil { + return fmt.Errorf("write plan: %w", err) + } + return nil + } + if _, err := fmt.Fprintln(out, "-- no changes to run; the live table carries an unfinished index build the plan leaves alone"); err != nil { return fmt.Errorf("write plan: %w", err) } - return nil + return writeAdvisoryText(out, report.Advisories) } if _, err := fmt.Fprintln(out, "-- plan derived by pg-sprite diff; execute statements via pg-sprite migrate,"); err != nil { return fmt.Errorf("write plan: %w", err) @@ -127,6 +137,18 @@ func writePlanText(out io.Writer, report plan.Report) error { return err } } + return writeAdvisoryText(out, report.Advisories) +} + +// writeAdvisoryText emits each plan advisory as an SQL comment: the typed +// kind, the index, and the kind's description. Comments only — an advisory +// is never a statement to run. +func writeAdvisoryText(out io.Writer, advisories []schemadiff.Advisory) error { + for _, a := range advisories { + if _, err := fmt.Fprintln(out, sqlDiagnosticComment(fmt.Sprintf("%s: index %q %s", a.Kind, a.Index, a.Kind.Description()))); err != nil { + return fmt.Errorf("write plan: %w", err) + } + } return nil } diff --git a/internal/cli/diff_text.go b/internal/cli/diff_text.go index 59e75a0d..783a14b8 100644 --- a/internal/cli/diff_text.go +++ b/internal/cli/diff_text.go @@ -6,8 +6,14 @@ import ( "github.com/block/pg-sprite/pkg/plan" "github.com/block/pg-sprite/pkg/router" + "github.com/block/pg-sprite/pkg/schemadiff" ) +// invalidIndexRunbookURL is where an advisory about an unfinished index +// build sends the operator: the runbook that proves an entry abandoned +// before removing it. +const invalidIndexRunbookURL = "https://github.com/block/pg-sprite/blob/main/docs/invalid-index-recovery.md" + // writeDiffText renders the derived convergence plan in the same // compiler-diagnostic grammar as the dry-run report: one labeled entry per // finding, the typed reason as the rule code, doc anchors, and a closing @@ -22,7 +28,10 @@ import ( // greenfield statement the create path refused by shape carries the typed // cause on the plan, and the renderer prints its description as a trailing // note on the refused statement — the executor's explanation, which the -// typed reason alone does not carry. +// typed reason alone does not carry. The plan's advisories — the invalid +// indexes the diff observed and declined to drop — render as warnings after +// the statements, with the typed kind as the rule code, so an empty plan +// over a table carrying debris never reads as converged. func writeDiffText(out io.Writer, pal palette, report plan.Report) error { w := &stickyWriter{out: out, pal: pal} if tableMissing(report) { @@ -33,8 +42,14 @@ func writeDiffText(out io.Writer, pal palette, report plan.Report) error { } } if len(report.Statements) == 0 { + writeAdvisoryDiagnostics(w, report.Advisories) w.entry("plan:") - w.printf(" %s — no changes; the live table matches the desired schema\n", targetText(report)) + if len(report.Advisories) == 0 { + w.printf(" %s — no changes; the live table matches the desired schema\n", targetText(report)) + } else { + w.printf(" %s — no changes to run; the live table carries %s the plan leaves alone\n", + targetText(report), countNoun(len(report.Advisories), "unfinished index build")) + } return w.err } steps, refused := 0, 0 @@ -46,6 +61,7 @@ func writeDiffText(out io.Writer, pal palette, report plan.Report) error { w.diag("note", "", "the create path refuses this statement: "+ps.Cause.Description()) } } + writeAdvisoryDiagnostics(w, report.Advisories) w.entry("plan:") w.printf(" %s — %s, %s to run, %d refused\n", targetText(report), countNoun(len(report.Statements), "statement"), countNoun(steps, "step"), refused) @@ -65,6 +81,21 @@ func writeDiffText(out io.Writer, pal palette, report plan.Report) error { return w.err } +// writeAdvisoryDiagnostics renders each plan advisory as a warning whose +// rule code is the typed kind, followed by one docs pointer at the runbook. +// A warning, not an error: the plan stays executable and the diff's exit +// code is unchanged — the operator is being told what the plan will not do. +func writeAdvisoryDiagnostics(w *stickyWriter, advisories []schemadiff.Advisory) { + if len(advisories) == 0 { + return + } + for _, a := range advisories { + w.diag("warning", string(a.Kind), fmt.Sprintf("index %q %s", a.Index, a.Kind.Description())) + } + w.entry("docs:") + w.printf(" %s\n", invalidIndexRunbookURL) +} + // diffRefused reports whether the derived plan contains any statement // execution would refuse. The diff exits with the refusal code in that // case — the same contract the dry run uses — so CI can gate on the diff diff --git a/internal/cli/diff_text_test.go b/internal/cli/diff_text_test.go index ca82187c..4cd794fe 100644 --- a/internal/cli/diff_text_test.go +++ b/internal/cli/diff_text_test.go @@ -275,6 +275,121 @@ func TestDiffTextNoChanges(t *testing.T) { `, out.String()) } +// An empty plan over a table carrying an invalid index the diff declined to +// drop must not read as converged: the advisory renders as a warning coded +// with its typed kind, the docs entry points at the runbook, and the plan +// line says what the table carries instead of claiming a match. +func TestDiffTextEmptyPlanWithAdvisoryIsNotConverged(t *testing.T) { + report := plan.NewReport(plan.SourceDiff) + report.Schema, report.Table, report.ServerVersion = "public", "users", "16.10" + report.Disposition = router.DispositionExecute + exists := true + report.TableExists = &exists + report.Advisories = []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexUndeclared, Index: "users_email_idx_ccnew"}, + } + + var out strings.Builder + require.NoError(t, writeDiffText(&out, palette{}, report)) + text := out.String() + assert.True(t, strings.HasPrefix(text, "warning[invalid-index-undeclared]:\n index \"users_email_idx_ccnew\" is an unfinished concurrent build"), + "the advisory leads with its typed kind as the rule code: %s", text) + assert.Contains(t, text, "docs:\n "+invalidIndexRunbookURL+"\n") + assert.Contains(t, text, "plan:\n public.users (PostgreSQL 16.10) — no changes to run; the live table carries 1 unfinished index build the plan leaves alone\n") + assert.NotContains(t, text, "matches the desired schema") + assert.NotContains(t, text, "apply:", "nothing to run, so no apply pointer") +} + +// A rebuild advisory rides alongside the create it explains: the statement +// renders as usual, and the warning after it says the create completes +// only through the proven-removal path. The plan stays executable — the +// warning changes neither the counts nor the apply pointer. +func TestDiffTextRebuildAdvisoryFollowsStatements(t *testing.T) { + report := plan.NewReport(plan.SourceDiff) + report.Schema, report.Table, report.ServerVersion = "public", "users", "16.10" + report.Disposition = router.DispositionExecute + exists := true + report.TableExists = &exists + sql := `CREATE UNIQUE INDEX "users_email_key" ON "public"."users" USING btree ("email")` + report.Statements = append(report.Statements, plan.Statement{ + SQL: sql, + Kind: schemadiff.ChangeCreateIndex, + Route: planner.RouteNative, + Backend: router.BackendNative, + Disposition: router.DispositionExecute, + Decisions: []planner.Decision{{ + Operation: "create index", + Route: planner.RouteNative, + Reason: planner.ReasonSaferIdiom, + }}, + ExecSQL: []string{`CREATE UNIQUE INDEX CONCURRENTLY "users_email_key" ON "public"."users" USING btree ("email")`}, + Execution: planner.ExecutionAutocommit, + }) + report.Advisories = []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexRebuild, Index: "users_email_key"}, + } + + var out strings.Builder + require.NoError(t, writeDiffText(&out, palette{}, report)) + text := out.String() + statementAt := strings.Index(text, "statement 1:\n "+sql+";\n") + advisoryAt := strings.Index(text, "warning[invalid-index-rebuild]:\n index \"users_email_key\" is an unfinished concurrent build") + planAt := strings.Index(text, "plan:\n public.users (PostgreSQL 16.10) — 1 statement, 1 step to run, 0 refused\n") + require.NotEqual(t, -1, statementAt, text) + require.NotEqual(t, -1, advisoryAt, text) + require.NotEqual(t, -1, planAt, text) + assert.Less(t, statementAt, advisoryAt, "the advisory follows the statements it explains") + assert.Less(t, advisoryAt, planAt, "the advisory precedes the plan summary") + assert.Contains(t, text, "docs:\n "+invalidIndexRunbookURL+"\n") + assert.Contains(t, text, "apply:", "the plan is still executable") +} + +// The --sql script carries advisories as comment lines only, and an empty +// plan with an advisory does not claim the table matches. +func TestPlanTextCommentsAdvisories(t *testing.T) { + report := plan.NewReport(plan.SourceDiff) + report.Schema, report.Table, report.ServerVersion = "public", "users", "16.10" + report.Disposition = router.DispositionExecute + exists := true + report.TableExists = &exists + report.Advisories = []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexUndeclared, Index: "users_email_idx_ccnew"}, + } + + var out strings.Builder + require.NoError(t, writePlanText(&out, report)) + text := out.String() + assert.Equal(t, "-- no changes to run; the live table carries an unfinished index build the plan leaves alone\n"+ + "-- invalid-index-undeclared: index \"users_email_idx_ccnew\" "+schemadiff.AdvisoryInvalidIndexUndeclared.Description()+"\n", text) + + sql := `CREATE INDEX "users_email_idx" ON "public"."users" USING btree ("email")` + report.Statements = append(report.Statements, plan.Statement{ + SQL: sql, + Kind: schemadiff.ChangeCreateIndex, + Route: planner.RouteNative, + Backend: router.BackendNative, + Disposition: router.DispositionExecute, + ExecSQL: []string{sql}, + Execution: planner.ExecutionAutocommit, + Decisions: []planner.Decision{{ + Operation: "create index", + Route: planner.RouteNative, + Reason: planner.ReasonMetadataOnly, + }}, + }) + report.Advisories[0] = schemadiff.Advisory{Kind: schemadiff.AdvisoryInvalidIndexRebuild, Index: "users_email_idx"} + out.Reset() + require.NoError(t, writePlanText(&out, report)) + text = out.String() + assert.True(t, strings.HasSuffix(text, sql+";\n-- invalid-index-rebuild: index \"users_email_idx\" "+schemadiff.AdvisoryInvalidIndexRebuild.Description()+"\n"), text) + for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") { + if strings.HasPrefix(line, "--") { + continue + } + assert.Equal(t, sql+";", line, "the only bare statement is the planned create") + } +} + // A plan containing a statement this build cannot execute renders the same // leading error diagnostic as the dry run, counts it refused, and drops the // apply pointer — there is nothing migrate would run. diff --git a/pkg/capabilities/capabilities.yaml b/pkg/capabilities/capabilities.yaml index 068ffda1..ff9c3b32 100644 --- a/pkg/capabilities/capabilities.yaml +++ b/pkg/capabilities/capabilities.yaml @@ -362,7 +362,7 @@ rows: front_doors: migrate: supported diff: supported - reason_notes: "`diff`, `pull`, and desired-file rendering round-trip the canonical model. The model carries each index's validity (`pg_index.indisvalid`): on a plain table an invalid entry is a concurrent build that did not finish — abandoned, or still running — so a live entry with the desired name and definition does not deliver the desired index, and `diff` plans it as a `create-index` change. `diff` never emits a drop for an invalid entry, whatever the desired file says about its name: a plain `DROP INDEX` blocks the table and cannot tell abandoned debris from a build still in progress. A `pull`ed baseline of such a table therefore re-diffs to that one rebuild rather than to zero. On a partitioned parent an invalid index means unattached partition indexes, not an unfinished build, so validity plays no part in the parent's comparison" + reason_notes: "`diff`, `pull`, and desired-file rendering round-trip the canonical model. The model carries each index's validity (`pg_index.indisvalid`): on a plain table an invalid entry is a concurrent build that did not finish — abandoned, or still running — so a live entry with the desired name and definition does not deliver the desired index, and `diff` plans it as a `create-index` change. `diff` never emits a drop for an invalid entry, whatever the desired file says about its name: a plain `DROP INDEX` blocks the table and cannot tell abandoned debris from a build still in progress. Every invalid entry the diff observes and leaves in place is named on the plan report as an advisory (`invalid-index-rebuild` when the desired file names it, `invalid-index-undeclared` when it does not), outside the statements and the fingerprint, so an empty plan over such a table reads as 'no changes to run' rather than converged. A `pull`ed baseline of such a table therefore re-diffs to that one rebuild rather than to zero. On a partitioned parent an invalid index means unattached partition indexes, not an unfinished build, so validity plays no part in the parent's comparison" - id: declarative-not-valid-constraint area: "declarative_model" operation: "A live check constraint or foreign key left `NOT VALID`" diff --git a/pkg/diffplan/diffplan.go b/pkg/diffplan/diffplan.go index d4fd754e..a25bd2e0 100644 --- a/pkg/diffplan/diffplan.go +++ b/pkg/diffplan/diffplan.go @@ -48,7 +48,9 @@ type Request struct { // desired schema against the live database: introspect the live table, // derive the changes (the full qualified desired schema when the table does // not exist yet), classify each change with live-column facts, route the -// set, and stamp the report with the server version and fingerprint. +// set, attach the diff's advisories (the invalid indexes it observed and +// declined to drop), and stamp the report with the server version and +// fingerprint. // // Plan needs more than a read-only connection: desired state is realized by // executing the desired DDL in a scratch schema inside a transaction that @@ -97,6 +99,10 @@ func Plan(ctx context.Context, pool *pgxpool.Pool, req Request) (plan.Report, er if changes, err = schemadiff.Diff(req.Schema, live, desired); err != nil { return plan.Report{}, err } + // The diff never drops an unfinished index build; the report + // carries each one it saw so an empty or non-destructive plan does + // not read as a clean table. + report.Advisories = schemadiff.Advisories(live, desired) if facts.AddColumnProofs, err = addColumnProofs(ctx, pool, req.Schema, ds.Table(), changes); err != nil { return plan.Report{}, err } diff --git a/pkg/diffplan/diffplan_integration_test.go b/pkg/diffplan/diffplan_integration_test.go index b8af5430..54d77d65 100644 --- a/pkg/diffplan/diffplan_integration_test.go +++ b/pkg/diffplan/diffplan_integration_test.go @@ -5,6 +5,7 @@ import ( "slices" "testing" + "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -236,6 +237,92 @@ func TestPlanNoChangesEmptyPlan(t *testing.T) { assert.Empty(t, report.Statements) } +// seedAbandonedUniqueBuild creates schema.events with two rows sharing a +// name and runs a unique concurrent build over them. The build fails on the +// duplicates after creating its catalog entry, which leaves +// events_name_key behind as an invalid index — the leftover of an +// unfinished concurrent build. +func seedAbandonedUniqueBuild(t *testing.T, pool *pgxpool.Pool, schema string) { + t.Helper() + _, err := pool.Exec(t.Context(), fmt.Sprintf(` +CREATE TABLE %[1]s.events ( + id bigint PRIMARY KEY, + name text NOT NULL +); +INSERT INTO %[1]s.events VALUES (1, 'dup'), (2, 'dup'); +`, schema)) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), fmt.Sprintf(` +CREATE UNIQUE INDEX CONCURRENTLY events_name_key ON %s.events (name) +`, schema)) + require.Error(t, err, "a unique build over duplicates must fail after creating its catalog entry") +} + +// A desired schema that no longer names the leftover of a failed concurrent +// build plans zero statements — the diff never drops an unfinished build — +// and one invalid-index-undeclared advisory, so the empty plan does not +// read as a clean table. The advisory is explanatory: the fingerprint is +// the empty plan's. +func TestPlanUndeclaredInvalidIndexReportsAdvisory(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + seedAbandonedUniqueBuild(t, pool, schema) + + report, err := diffplan.Plan(t.Context(), pool, diffplan.Request{ + Schema: schema, + Desired: parseDesired(t, ` +CREATE TABLE events ( + id bigint PRIMARY KEY, + name text NOT NULL +); +`), + }) + require.NoError(t, err) + assert.Empty(t, report.Statements, "the diff never drops an unfinished build") + assert.Equal(t, router.DispositionExecute, report.Disposition) + assert.Equal(t, []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexUndeclared, Index: "events_name_key"}, + }, report.Advisories) + assert.Equal(t, plan.Fingerprint(nil), report.Fingerprint, "an advisory is not part of the plan's identity") +} + +// A desired schema that names the leftover of a failed concurrent build +// plans its non-destructive create-index and an invalid-index-rebuild +// advisory: the create cannot run as-is against the occupied name, and the +// plan completes only through the proven-removal path. The advisory changes +// neither the destructive gate nor the fingerprint. +func TestPlanDeclaredInvalidIndexReportsRebuildAdvisory(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + seedAbandonedUniqueBuild(t, pool, schema) + + report, err := diffplan.Plan(t.Context(), pool, diffplan.Request{ + Schema: schema, + Desired: parseDesired(t, ` +CREATE TABLE events ( + id bigint PRIMARY KEY, + name text NOT NULL +); + +CREATE UNIQUE INDEX events_name_key ON events (name); +`), + }) + require.NoError(t, err) + require.Len(t, report.Statements, 1) + assert.Equal(t, schemadiff.ChangeCreateIndex, report.Statements[0].Kind) + assert.False(t, report.Statements[0].Destructive, "the advisory does not turn the create destructive") + assert.Equal(t, []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexRebuild, Index: "events_name_key"}, + }, report.Advisories) + assert.Equal(t, plan.Fingerprint(report.Statements), report.Fingerprint) +} + // A table that does not exist yet plans as the full desired schema, // qualified onto the target schema. func TestPlanMissingTableEmitsFullDesiredSchema(t *testing.T) { diff --git a/pkg/plan/docs_test.go b/pkg/plan/docs_test.go index 1b61f7bd..f3d976b3 100644 --- a/pkg/plan/docs_test.go +++ b/pkg/plan/docs_test.go @@ -156,6 +156,9 @@ func TestDocListsEveryVocabularyValue(t *testing.T) { for _, k := range schemadiff.ChangeKinds() { values = append(values, string(k)) } + for _, k := range schemadiff.AdvisoryKinds() { + values = append(values, string(k)) + } for _, g := range suggest.Guidances() { values = append(values, string(g)) } diff --git a/pkg/plan/plan.go b/pkg/plan/plan.go index 89d3660a..7775e6c6 100644 --- a/pkg/plan/plan.go +++ b/pkg/plan/plan.go @@ -37,12 +37,14 @@ import ( // the cascading-drop planner reason. Version 12 routes an app-breaking // rename to the rewrite-required disposition and added the // expand-contract-column and coordinated-table-rename guidance codes. -// Version 14 added the alter-column-type-in-place guidance code, which a -// destructive-change refusal verdict carries; no report emits it, but the -// report embeds the whole guidance vocabulary. Version 13 classifies the -// rename of an index that backs a constraint as app-breaking-rename and -// added the coordinated-constraint-rename guidance code. -const FormatVersion = 14 +// Version 13 classifies the rename of an index that backs a constraint as +// app-breaking-rename and added the coordinated-constraint-rename guidance +// code. Version 14 added the alter-column-type-in-place guidance code, which +// a destructive-change refusal verdict carries; no report emits it, but the +// report embeds the whole guidance vocabulary. Version 15 added the +// report-level advisories field and its closed kind vocabulary for the +// invalid indexes a diff observes and declines to drop. +const FormatVersion = 15 // Source identifies which front door derived the plan. type Source string @@ -189,6 +191,15 @@ type Report struct { Fingerprint string `json:"fingerprint"` // Statements is the ordered plan; empty means there is nothing to do. Statements []Statement `json:"statements"` + // Advisories are the typed observations the diff made about the live + // table that produced no statement (schemadiff.Advisory): the invalid + // indexes it saw and declined to drop. Present only on a diff-source + // plan whose table exists, and only when there is something to report. + // Advisory, not executable — the change list and the destructive gate + // are unaffected, and an empty Statements with an advisory present + // means the table is not clean even though there is nothing to run. + // Explanatory, so excluded from the fingerprint. + Advisories []schemadiff.Advisory `json:"advisories,omitempty"` } // RefuseUnsupportedPartitionedParent marks statements as refused when diff --git a/pkg/plan/plan_test.go b/pkg/plan/plan_test.go index 9c661e0f..d41dc896 100644 --- a/pkg/plan/plan_test.go +++ b/pkg/plan/plan_test.go @@ -478,7 +478,7 @@ func TestDiscloseGreenfieldExecutionRequiresAbsentTable(t *testing.T) { } // The JSON shape is the adapter-facing contract: exact keys, exact -// omissions. A consumer pins format_version 11 against this test. +// omissions. A consumer pins format_version 15 against this test. func TestReportJSONShape(t *testing.T) { exists := true r := plan.Report{ @@ -534,7 +534,7 @@ func TestReportJSONShape(t *testing.T) { raw, err := json.Marshal(r) require.NoError(t, err) assert.JSONEq(t, fmt.Sprintf(`{ - "format_version": 14, + "format_version": 15, "source": "diff", "schema": "public", "table": "t", @@ -592,7 +592,7 @@ func TestReportJSONOmitsUnsetOptionalFields(t *testing.T) { raw, err := json.Marshal(r) require.NoError(t, err) assert.JSONEq(t, `{ - "format_version": 14, + "format_version": 15, "source": "alter", "disposition": "execute", "fingerprint": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -600,6 +600,38 @@ func TestReportJSONOmitsUnsetOptionalFields(t *testing.T) { }`, string(raw)) } +// The report-level advisories ride alongside an empty statement list with +// exact keys: a consumer reading "nothing to run" must also see what the +// diff declined to do. Absent — not an empty array — when there is nothing +// to report (TestReportJSONOmitsUnsetOptionalFields). +func TestReportJSONAdvisoriesShape(t *testing.T) { + exists := true + r := plan.NewReport(plan.SourceDiff) + r.Schema, r.Table, r.ServerVersion = "public", "t", "16.4" + r.TableExists = &exists + r.Disposition = router.DispositionExecute + r.Advisories = []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexUndeclared, Index: "t_c_idx_ccnew"}, + } + r.Fingerprint = plan.Fingerprint(r.Statements) + raw, err := json.Marshal(r) + require.NoError(t, err) + assert.JSONEq(t, `{ + "format_version": 15, + "source": "diff", + "schema": "public", + "table": "t", + "server_version": "16.4", + "table_exists": true, + "disposition": "execute", + "fingerprint": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "statements": [], + "advisories": [ + {"kind": "invalid-index-undeclared", "index": "t_c_idx_ccnew"} + ] + }`, string(raw)) +} + // The statement-level cause is emitted only when a greenfield create-shape // refusal set it; every other statement omits the key rather than // serializing an empty string. @@ -676,7 +708,7 @@ func TestFingerprintCoversExecutionNotExplanation(t *testing.T) { } // Sources is the closed vocabulary a consumer branches on; the set is -// pinned to format_version 11. +// pinned to format_version 15. func TestSourcesVocabularyPinned(t *testing.T) { assert.Equal(t, []plan.Source{plan.SourceAlter, plan.SourceDiff}, plan.Sources()) } diff --git a/pkg/schemadiff/advisory.go b/pkg/schemadiff/advisory.go new file mode 100644 index 00000000..9ace5dd6 --- /dev/null +++ b/pkg/schemadiff/advisory.go @@ -0,0 +1,81 @@ +package schemadiff + +// AdvisoryKind classifies an observation the diff made but could not turn +// into a change, so a consumer can act on it without parsing prose. +type AdvisoryKind string + +// The advisory kinds a plan can carry. Both describe an invalid index on a +// plain table — an unfinished concurrent build, abandoned or still running +// — which the diff never drops (see indexNeedsDrop). +const ( + // AdvisoryInvalidIndexRebuild marks an invalid index the desired + // schema names, with the same or a different definition: the plan + // carries its create-index, and that create cannot run as-is against + // the occupied name. Completing the plan goes through the proven + // removal path (executor.RebuildAbandonedIndex or the invalid-index + // runbook), which renames and drops the entry only after proving it + // abandoned. + AdvisoryInvalidIndexRebuild AdvisoryKind = "invalid-index-rebuild" + // AdvisoryInvalidIndexUndeclared marks an invalid index the desired + // schema does not name: the diff plans no drop for it, so no statement + // touches it and an empty plan is not evidence the table is clean. An + // abandoned entry clears per the invalid-index runbook; a + // build still running finishes on its own. + AdvisoryInvalidIndexUndeclared AdvisoryKind = "invalid-index-undeclared" +) + +// AdvisoryKinds returns the closed set of AdvisoryKind values. It is part +// of the plan-report contract (docs/plan-report.md): the set changes only +// with a format_version bump, and a consumer that meets an unrecognized +// value must treat the advisory as unknown and surface it. +func AdvisoryKinds() []AdvisoryKind { + return []AdvisoryKind{ + AdvisoryInvalidIndexRebuild, + AdvisoryInvalidIndexUndeclared, + } +} + +// Description renders the kind for an operator: what the diff observed and +// what completes the plan. Display only — consumers branch on the kind. +func (k AdvisoryKind) Description() string { + switch k { + case AdvisoryInvalidIndexRebuild: + return "is an unfinished concurrent build (invalid) that the desired schema names; the planned create cannot run against the occupied name, and the plan completes only through the proven-removal path (executor.RebuildAbandonedIndex or the invalid-index runbook), which drops the entry after proving it abandoned" + case AdvisoryInvalidIndexUndeclared: + return "is an unfinished concurrent build (invalid) that the desired schema does not name; the diff never drops it, so no planned statement touches it and an empty plan does not mean the table is clean — clear an abandoned entry per the invalid-index runbook, or wait for a build still running to finish" + } + return string(k) +} + +// Advisory is one typed observation the diff made about the live table +// that produced no change: reported so a consumer sees what the plan +// declined to do, never acted on by the engine. The table is the report's. +type Advisory struct { + // Kind classifies the observation. + Kind AdvisoryKind `json:"kind"` + // Index is the live index the observation is about. + Index string `json:"index"` +} + +// Advisories reports every invalid index on a plain live table, in live +// index order, classified by whether desired names it. These are the +// entries Diff observes and declines to drop; Diff's change list alone +// would discard the observation, so a caller reporting a plan pairs the two +// over the same models. A partitioned parent's invalid entries are not +// unfinished builds and produce no advisory. +func Advisories(live, desired Model) []Advisory { + partitioned := live.PartitionKey != "" + desiredIdx := indexesByName(desired.Indexes) + var advisories []Advisory + for _, ix := range live.Indexes { + if !isUnfinishedBuild(partitioned, ix) { + continue + } + kind := AdvisoryInvalidIndexUndeclared + if _, named := desiredIdx[ix.Name]; named { + kind = AdvisoryInvalidIndexRebuild + } + advisories = append(advisories, Advisory{Kind: kind, Index: ix.Name}) + } + return advisories +} diff --git a/pkg/schemadiff/advisory_test.go b/pkg/schemadiff/advisory_test.go new file mode 100644 index 00000000..cb8535dc --- /dev/null +++ b/pkg/schemadiff/advisory_test.go @@ -0,0 +1,107 @@ +package schemadiff + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A valid live table produces no advisory: advisories exist only for the +// observations the diff declines to act on. +func TestAdvisoriesNoneForValidIndexes(t *testing.T) { + assert.Empty(t, Advisories(base(), base())) +} + +// The invalid entry desired names with the same definition is the rebuild +// case: the diff plans the create, and the advisory says the create +// completes only through the proven-removal path. +func TestAdvisoriesInvalidIndexNamedByDesiredIsRebuild(t *testing.T) { + live := base() + live.Indexes[0].Invalid = true + assert.Equal(t, []Advisory{ + {Kind: AdvisoryInvalidIndexRebuild, Index: "events_name_idx"}, + }, Advisories(live, base())) +} + +// A redefined invalid entry is still the rebuild case: desired names the +// index, so the create is planned and the same completion path applies. +func TestAdvisoriesInvalidIndexRedefinedIsRebuild(t *testing.T) { + live := base() + live.Indexes[0].Invalid = true + desired := base() + desired.Indexes[0].Def = "CREATE UNIQUE INDEX events_name_idx ON events USING btree (name)" + assert.Equal(t, []Advisory{ + {Kind: AdvisoryInvalidIndexRebuild, Index: "events_name_idx"}, + }, Advisories(live, desired)) +} + +// An invalid entry desired no longer names yields no change and one +// undeclared advisory: the observation the change list would discard. A +// valid entry in the same position is dropped as usual and produces no +// advisory. +func TestAdvisoriesInvalidIndexAbsentFromDesiredIsUndeclared(t *testing.T) { + live := base() + live.Indexes = append(live.Indexes, + Index{Name: "events_name_idx_ccnew", Def: "CREATE INDEX events_name_idx_ccnew ON events USING btree (name)", Invalid: true}, + Index{Name: "events_legacy_idx", Def: "CREATE INDEX events_legacy_idx ON events USING btree (id)"}, + ) + desired := base() + desired.Indexes = nil + + changes, err := Diff("public", live, desired) + require.NoError(t, err) + assert.Equal(t, []string{ + `DROP INDEX "public"."events_name_idx"`, + `DROP INDEX "public"."events_legacy_idx"`, + }, sqls(changes), "the valid entries drop; the invalid one is left alone") + assert.Equal(t, []Advisory{ + {Kind: AdvisoryInvalidIndexUndeclared, Index: "events_name_idx_ccnew"}, + }, Advisories(live, desired)) +} + +// Advisories follow live index order and classify each entry independently. +func TestAdvisoriesReportEveryInvalidEntryInLiveOrder(t *testing.T) { + live := base() + live.Indexes[0].Invalid = true + live.Indexes = append(live.Indexes, + Index{Name: "events_name_idx_ccnew", Def: "CREATE INDEX events_name_idx_ccnew ON events USING btree (name)", Invalid: true}, + ) + assert.Equal(t, []Advisory{ + {Kind: AdvisoryInvalidIndexRebuild, Index: "events_name_idx"}, + {Kind: AdvisoryInvalidIndexUndeclared, Index: "events_name_idx_ccnew"}, + }, Advisories(live, base())) +} + +// On a partitioned parent an invalid index means unattached partition +// indexes, not an unfinished build, so it is never advised — whether or +// not desired names it. +func TestAdvisoriesNoneOnPartitionedParent(t *testing.T) { + live := base() + live.PartitionKey = "RANGE (id)" + live.Indexes[0].Invalid = true + desired := base() + desired.PartitionKey = "RANGE (id)" + assert.Empty(t, Advisories(live, desired)) + + desired.Indexes = nil + assert.Empty(t, Advisories(live, desired)) +} + +// The kind set is a closed vocabulary a consumer branches on; pinned to +// the plan report's format_version. +func TestAdvisoryKindsVocabularyPinned(t *testing.T) { + assert.Equal(t, []AdvisoryKind{ + AdvisoryInvalidIndexRebuild, + AdvisoryInvalidIndexUndeclared, + }, AdvisoryKinds()) +} + +// Every kind describes itself; an unknown kind falls back to its own token +// so a renderer never prints an empty note. +func TestAdvisoryKindDescriptions(t *testing.T) { + for _, k := range AdvisoryKinds() { + assert.NotEqual(t, string(k), k.Description(), "kind %q has no description", k) + } + assert.Equal(t, "no-such-kind", AdvisoryKind("no-such-kind").Description()) +} diff --git a/pkg/schemadiff/diff.go b/pkg/schemadiff/diff.go index 522f9301..f80f2cc9 100644 --- a/pkg/schemadiff/diff.go +++ b/pkg/schemadiff/diff.go @@ -158,7 +158,9 @@ type ConstraintValidation struct { // order for columns, name order for constraints and indexes. schema // qualifies the emitted statements' table references. Columns are compared // by name only: attribute order carries no semantics in PostgreSQL and is -// deliberately out of scope for convergence. +// deliberately out of scope for convergence. An invalid index on a plain +// table never yields a drop (see indexNeedsDrop); [Advisories] over the same +// models reports each such entry so the observation is not lost. // RowSecurity is observational only: table-only desired files do not claim // ownership of access control, so missing policies never produce policy drops. func Diff(schema string, live, desired Model) ([]Change, error) { diff --git a/pkg/schemadiff/schemadiff_integration_test.go b/pkg/schemadiff/schemadiff_integration_test.go index 915412f8..ae1f7a47 100644 --- a/pkg/schemadiff/schemadiff_integration_test.go +++ b/pkg/schemadiff/schemadiff_integration_test.go @@ -389,6 +389,9 @@ CREATE UNIQUE INDEX events_name_key ON events (name); assert.Equal(t, schemadiff.ChangeCreateIndex, changes[0].Kind) assert.False(t, changes[0].Destructive) assert.Equal(t, fmt.Sprintf("CREATE UNIQUE INDEX events_name_key ON %s.events USING btree (name)", schema), changes[0].SQL) + assert.Equal(t, []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexRebuild, Index: "events_name_key"}, + }, schemadiff.Advisories(live, want), "the non-destructive create is accompanied by the advisory that completing it removes the leftover") // A baseline pulled from the table renders the leftover like any other // index and materializes it valid, so the round trip diffs to exactly @@ -431,6 +434,7 @@ CREATE UNIQUE INDEX events_name_key ON events (name); rediff, err := schemadiff.Diff(schema, after, want) require.NoError(t, err) assert.Empty(t, rediff, "a valid index with the desired definition is delivered") + assert.Empty(t, schemadiff.Advisories(after, want), "a delivered index leaves nothing to advise") } // Desired may name the leftover of a failed build with a different @@ -477,6 +481,9 @@ CREATE INDEX events_name_key ON events (name); assert.Equal(t, schemadiff.ChangeCreateIndex, changes[0].Kind) assert.False(t, changes[0].Destructive) assert.Equal(t, fmt.Sprintf("CREATE INDEX events_name_key ON %s.events USING btree (name)", schema), changes[0].SQL) + assert.Equal(t, []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexRebuild, Index: "events_name_key"}, + }, schemadiff.Advisories(live, want), "a redefined leftover is still the rebuild case: desired names it") // The duplicate rows stay: they only ever conflicted with the abandoned // unique build, not with the plain index desired now. @@ -592,6 +599,9 @@ CREATE TABLE events ( changes, err = schemadiff.Diff(schema, live, without) require.NoError(t, err) assert.Empty(t, changes, "an in-flight build desired does not name is not dropped from under its builder") + assert.Equal(t, []schemadiff.Advisory{ + {Kind: schemadiff.AdvisoryInvalidIndexUndeclared, Index: "events_name_idx"}, + }, schemadiff.Advisories(live, without), "the empty plan carries the observation the change list discards") // Releasing the snapshot lets the build finish; the same desired state // then diffs to nothing.