From f069301fa84474564e1adcdcc92bee82836d95bf Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 26 Sep 2026 18:20:56 +0200 Subject: [PATCH 1/5] Load-time refusals are validation verdicts Every input-domained refusal raised while loading a bundle now becomes a ValidateBundleError item instead of a no-verdict fault. The unknown model is the worked example: the operator raises a located PipeOperatorModelChoiceError (input domain, caller-facing) that the cascade turns into a pipe_validation item with the unknown_model code, the field path, the reference as written, its model type and the deck's close matches, plus a rename-model fix when there is exactly one. Other input refusals at pipe build are wrapped in the new PipeLoadRefusalError, located on the pipe and its source. The bare and agent validate commands route their library load through the cascade, so they exit 1 with the invalid verdict, and a failing dry run under `pipelex validate pipe` or `--all` renders the panel rather than a traceback. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FigDssaJrvNcmbnBedi7oq --- .drift/acks/cli-docs.toml | 22 +- CHANGELOG.md | 12 + docs/contribute/mthds-test-corpus.md | 2 +- docs/errors/authoring-and-language.md | 1 + docs/errors/model-choice-not-found-error.md | 2 +- docs/errors/pipe-load-refusal-error.md | 21 ++ .../pipe-operator-model-choice-error.md | 4 +- docs/errors/validate-bundle-error.md | 2 +- docs/tools/cli/agent-cli.md | 3 + docs/tools/cli/validate.md | 23 ++ docs/under-the-hood/error-model.md | 8 +- pipelex/base_exceptions.py | 7 + pipelex/cli/agent_cli/CLAUDE.md | 1 + .../cli/agent_cli/commands/agent_output.py | 15 +- .../commands/validate/_validate_core.py | 19 +- .../agent_cli/commands/validate/bundle_cmd.py | 12 - .../agent_cli/commands/validate/method_cmd.py | 12 - .../agent_cli/commands/validate/pipe_cmd.py | 23 -- .../cli/commands/validate/_validate_core.py | 56 +-- pipelex/cli/error_handlers.py | 10 +- pipelex/cogt/exceptions.py | 6 +- pipelex/core/exceptions.py | 16 +- pipelex/core/pipes/exceptions.py | 124 +++++-- pipelex/libraries/library_manager.py | 52 ++- pipelex/mthds_parsing/handle_pipe_errors.py | 33 +- .../pipe_operators/extract/pipe_extract.py | 6 +- .../pipe_operators/img_gen/pipe_img_gen.py | 3 +- pipelex/pipe_operators/llm/pipe_llm.py | 11 +- pipelex/pipe_operators/pipe_operator.py | 27 +- pipelex/pipe_operators/search/pipe_search.py | 6 +- .../structure/pipe_structure.py | 3 +- pipelex/pipeline/exceptions.py | 6 +- pipelex/pipeline/fixes/planner.py | 38 ++- pipelex/pipeline/validate_bundle.py | 79 ++++- pipelex/pipeline/validation_errors.py | 3 + .../invalid_unknown_model/bundle.mthds | 12 + .../entries/invalid_unknown_model/entry.toml | 12 + .../test_extras/mthds_corpus/vocabulary.toml | 4 + pipelex/validation_error_types.py | 42 +++ tests/data/errors/error_identity.txt | 1 + .../cli/test_validate_load_refusals_cli.py | 174 ++++++++++ .../test_validate_bundle_load_refusals.py | 320 ++++++++++++++++++ tests/unit/pipelex/cli/test_agent_output.py | 2 +- tests/unit/pipelex/cli/test_error_handlers.py | 2 + .../cli/test_error_handlers_snapshot.py | 2 + .../unit/pipelex/cli/test_run_core_wrapper.py | 2 + .../pipeline/fixes/test_fix_planner.py | 54 ++- .../pipeline/test_refusal_verdict_arms.py | 129 +++++++ 48 files changed, 1260 insertions(+), 164 deletions(-) create mode 100644 docs/errors/pipe-load-refusal-error.md create mode 100644 pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/bundle.mthds create mode 100644 pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/entry.toml create mode 100644 tests/integration/pipelex/cli/test_validate_load_refusals_cli.py create mode 100644 tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py create mode 100644 tests/unit/pipelex/pipeline/test_refusal_verdict_arms.py diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml index 204eb8ffe5..6fc2adb012 100644 --- a/.drift/acks/cli-docs.toml +++ b/.drift/acks/cli-docs.toml @@ -1,8 +1,8 @@ contract = "cli-docs" -digest = "sha256:2ac30bcfcf3e7d174db15b59fa2df4e9cb64c5c6a2c64f1d9e8bef415a4efd8f" -reviewed_by = "Claude (Opus 5.5)" -reviewed_at = "2026-09-25T12:48:43Z" -rationale = "Merge of origin/feature/Log-sink-seam (carrying dev's releases v0.61.0 through v0.65.0) into feature/Log-redaction. Against the parent the only trigger change is this branch's doctor_cmd.py arm that keeps an installed sink which failed during the replay of the held records, and the checks.log_sink row in pipelex/cli/agent_cli/CLAUDE.md carries that fourth case in the merged tree. Against this branch's previous head the only trigger change is the parent's check_model_cmd.py argument-help example moving from the retired @best-claude alias to @best-gpt, and neither docs/tools/cli/ nor the agent-CLI contract doc quotes that example or any retired deck alias (a grep for best-claude, best-gemini and best-mistral over both targets finds nothing). docs/tools/cli/agent-cli.md describes the doctor command by its flags and formats only, so nothing there is stale. The ack file's merge spliced the parent's header onto this branch's trigger snapshot, so this re-ack records the merged tree as reviewed as a whole. No doc change needed." +digest = "sha256:f03ae0caaedf4e4b644af61e7ace8a4d9c08597e9b1a48906a9334daa8e5a27a" +reviewed_by = "Louis Choquel" +reviewed_at = "2026-09-26T16:11:26Z" +rationale = "The validate commands (bare and agent) now run their library load inside translate_to_validate_bundle_error, so a load-time refusal and the unknown model leave as a ValidateBundleError verdict with exit 1; the PipeOperatorModelChoiceError exit-2 arms and its config entry in AGENT_ERROR_DOMAINS are gone (the class declares its input domain). Reviewed docs/tools/cli/validate.md and agent-cli.md (updated with the verdict sections) and pipelex/cli/agent_cli/CLAUDE.md (new Validate refusals are verdicts bullet)." [trigger_files] "pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc" @@ -12,7 +12,7 @@ rationale = "Merge of origin/feature/Log-sink-seam (carrying dev's releases v0.6 "pipelex/cli/agent_cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" "pipelex/cli/agent_cli/commands/accept_gateway_terms_cmd.py" = "blob:cb8338c09a7b5dee6088551c156d69dab613213a" "pipelex/cli/agent_cli/commands/agent_cli_factory.py" = "blob:49b8bab889d8673e7d4f1d683efe0928e43914b9" -"pipelex/cli/agent_cli/commands/agent_output.py" = "blob:3dc0283ef1daee4e17183b01f8a5c9d92f2688af" +"pipelex/cli/agent_cli/commands/agent_output.py" = "blob:27f8c90f6a209ee780e06f58f630d7efa59227ab" "pipelex/cli/agent_cli/commands/bundle_path_resolver.py" = "blob:30339c5d91048f4e67312d5691ed9a65d57472e0" "pipelex/cli/agent_cli/commands/check_model_cmd.py" = "blob:91589e517509248a4639a37e82f46a69538f8071" "pipelex/cli/agent_cli/commands/codegen/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" @@ -47,11 +47,11 @@ rationale = "Merge of origin/feature/Log-sink-seam (carrying dev's releases v0.6 "pipelex/cli/agent_cli/commands/run/pipe_cmd.py" = "blob:df5e7f80a328932ffde901eda8d0585d009678ad" "pipelex/cli/agent_cli/commands/run/stdin_resolver.py" = "blob:c7c7d7495508f1c4e5a46eba336a4ab1694ebcf7" "pipelex/cli/agent_cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" -"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:df07de7f203f57041907e080bc5fd80f9f593301" +"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:de1e56549d9e036d218cfd308174082bab7bc800" "pipelex/cli/agent_cli/commands/validate/app.py" = "blob:d2d0ed19288c2d1c25be1650dd962189fcdb3b86" -"pipelex/cli/agent_cli/commands/validate/bundle_cmd.py" = "blob:27cc2c0b0a7fb5d3a232c6c3ce44663572e14260" -"pipelex/cli/agent_cli/commands/validate/method_cmd.py" = "blob:794914167c13975bc0cdff7b301a8268ff1b79b6" -"pipelex/cli/agent_cli/commands/validate/pipe_cmd.py" = "blob:2f7dce6a1d3d9987aa14146bac79d6b0e7e985e1" +"pipelex/cli/agent_cli/commands/validate/bundle_cmd.py" = "blob:fb4aab0b8e4e381bb0c44cb07308987938d69e41" +"pipelex/cli/agent_cli/commands/validate/method_cmd.py" = "blob:cf841e73d6af1a7e90d5ddfadaad012928f1a3ba" +"pipelex/cli/agent_cli/commands/validate/pipe_cmd.py" = "blob:36f660819d85bca46bc4348f4b3e7a75fb393d19" "pipelex/cli/bundle_target_resolution.py" = "blob:8dccc50b700b877968ed1b2c1c3652ab1c32e956" "pipelex/cli/cli_factory.py" = "blob:1f859ab3b995e1a67e5fdc812a168d56c338e970" "pipelex/cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" @@ -120,14 +120,14 @@ rationale = "Merge of origin/feature/Log-sink-seam (carrying dev's releases v0.6 "pipelex/cli/commands/show_cmd.py" = "blob:9492c9af966877f694ca61f7b1db3d151a1ddf56" "pipelex/cli/commands/update_cmd.py" = "blob:06100ec3b9d6c66ca89a8c1536aa27983ae409ca" "pipelex/cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" -"pipelex/cli/commands/validate/_validate_core.py" = "blob:3294c9c716ac616a89213c7356d38a8eff62da40" +"pipelex/cli/commands/validate/_validate_core.py" = "blob:ef7e10657b5a9ffd84f8cf01a4ffa76ead726da7" "pipelex/cli/commands/validate/app.py" = "blob:309061423af13f2f6c496a024306ad696af567ab" "pipelex/cli/commands/validate/bundle_cmd.py" = "blob:1fbf3e0cf695db39101c83160a724df6468988e9" "pipelex/cli/commands/validate/method_cmd.py" = "blob:f12ee233fc985d04a4512f1fef6b9ea02a1b9644" "pipelex/cli/commands/validate/pipe_cmd.py" = "blob:d67267571559006459628180f2c8b216ba7fee31" "pipelex/cli/commands/which_cmd.py" = "blob:68661d4bab22e2c4dab183d59bd9f2c6dd573472" "pipelex/cli/deck_notice.py" = "blob:542877125e89e35b02bc969cfee952155b0d7c2c" -"pipelex/cli/error_handlers.py" = "blob:41f33d7eae4d583f380f3ec34637733b0ca9d840" +"pipelex/cli/error_handlers.py" = "blob:b0fb1813633f380a0c4e4f1ea2b2f5b2d57b2c23" "pipelex/cli/exceptions.py" = "blob:1d51c3a53ddd3f6cf8de566fa2ade3fa75ab7a3a" "pipelex/cli/installed_methods.py" = "blob:09197f47baf8d7251d28d148a0544b4a23ee7365" "pipelex/cli/method_resolver.py" = "blob:1230dde5f89ac28bbdcbcbbd0c1d485c71803caf" diff --git a/CHANGELOG.md b/CHANGELOG.md index b683372c51..988b5938ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [Unreleased] + +### Added + +- **An unknown model is a located `unknown_model` validation item**: a pipe whose model field names a handle, alias, preset or waterfall the model deck does not define now validates to an invalid verdict with one `pipe_validation` item of the new closed `error_type` `unknown_model`, carrying the `pipe_code`, `domain_code`, `source`, `field_name`, the `field_path` of the reference (`pipe..model`, or `pipe..model_to_structure`), and the new optional item fields `model_reference` (the reference as written), `model_type` and `suggestions` (the deck's close matches of the same kind, also kept in the message). Every pipe type that names a model gives the same item, where `PipeExtract` and `PipeSearch` used to give an `unknown_validation_error` whose message was a Python repr, and with exactly one suggestion the item carries a safe `rename-model` fix that `pipelex fix bundle` applies. The MTHDS Test Corpus gains the `invalid_unknown_model` entry covering the new `error.unknown_model` tag. + +### Changed + +- **Every refusal raised while loading a bundle is a validation verdict (Breaking)**: validation turns any refusal of the caller's input that has no arm of its own into an invalid verdict with one item, keeping the refusal's message only when it is caller-facing and otherwise its title, while a configuration or runtime fault still propagates as no verdict. A refusal raised while a pipe is built leaves the library load as the new `PipeLoadRefusalError`, naming the pipe, its domain and its file with the original as its cause, and gives a `pipe_validation` item located there; any other gives a `blueprint_validation` item. `pipelex validate bundle` no longer prints a traceback for an unknown model, `pipelex-agent validate bundle` answers it with `is_valid: false` and exit 1 instead of the no-verdict envelope and exit 2, and the in-process validator behind `/validate` returns it as an invalid verdict instead of raising. +- **`PipeOperatorModelChoiceError` is the located unknown-model refusal (Breaking)**: it is now raised whenever a pipe naming an unknown model is built, including on the run path, which still refuses the bundle before any pipe runs with an HTTP 422 whose message now names the pipe and the field; it is `input`-domained and caller-facing, carries `domain_code`, `field_name`, `suggestions` and `source`, and its `model_choice` is the reference as written. The agent CLI's hint for it points at `check-model` rather than `doctor`. +- **`pipelex validate pipe` and `pipelex validate --all` render a refusal as an invalid bundle**: a pipe whose dry run fails, and a refusal of the libraries while they load, now print the grouped invalid-bundle output and exit 1 instead of a traceback; `pipelex-agent validate pipe` and `--all` answer a load-time refusal with the invalid-verdict envelope. + ## [v0.66.0] - 2026-09-25 ### Highlights diff --git a/docs/contribute/mthds-test-corpus.md b/docs/contribute/mthds-test-corpus.md index b8e114d894..45aca9517f 100644 --- a/docs/contribute/mthds-test-corpus.md +++ b/docs/contribute/mthds-test-corpus.md @@ -48,7 +48,7 @@ An entry directory holds either exactly one `.mthds` file, or several with a `bu ] ``` -4. **Name no model.** Presets and aliases are resolved by the validation engine, not only at run time, so an entry pinning one fails validation outright on any consumer whose deck does not define it — which turns an entry about a language feature into an entry about model selection. Leave the choice to each consumer's deck. +4. **Name no model.** Presets and aliases are resolved by the validation engine, not only at run time, so an entry pinning one fails validation outright on any consumer whose deck does not define it — which turns an entry about a language feature into an entry about model selection. Leave the choice to each consumer's deck. The one entry that names a model is `invalid_unknown_model`, whose defect is exactly that: it names a handle no deck defines, so it fails with `unknown_model` on every consumer. 5. **Validate it locally** — against the local runtime, never the hosted API, which lags it: diff --git a/docs/errors/authoring-and-language.md b/docs/errors/authoring-and-language.md index 6b20b81074..2e89f551a6 100644 --- a/docs/errors/authoring-and-language.md +++ b/docs/errors/authoring-and-language.md @@ -42,6 +42,7 @@ own page. Classes are grouped by subsystem. - [`PipeFactoryError`](pipe-factory-error.md) — Pipe factory - [`PipeInputError`](pipe-input-error.md) — Pipe input - [`PipeInputsFactoryError`](pipe-inputs-factory-error.md) — Pipe inputs factory +- [`PipeLoadRefusalError`](pipe-load-refusal-error.md) — Pipe load refusal - [`PipeOperatorModelChoiceError`](pipe-operator-model-choice-error.md) — Pipe operator model choice - [`PipeRunError`](pipe-run-error.md) — Pipe run - [`PipeRunInputsError`](pipe-run-inputs-error.md) — Pipe run inputs diff --git a/docs/errors/model-choice-not-found-error.md b/docs/errors/model-choice-not-found-error.md index 39179d80de..2e58510190 100644 --- a/docs/errors/model-choice-not-found-error.md +++ b/docs/errors/model-choice-not-found-error.md @@ -7,7 +7,7 @@ description: "Reference for the `ModelChoiceNotFoundError` Pipelex error class." # Model choice not found -Error raised when a model choice cannot be found in the model deck. +Raised when a model reference names a handle, alias, preset or waterfall the model deck does not define: by the deck check a pipe runs when it is built, and by the deck when a run resolves a reference. When a pipe is built, the pipe operator raises it again as a ``PipeOperatorModelChoiceError`` located on the pipe and the field, so a bundle naming an unknown model is an invalid validation verdict (error type ``unknown_model``), never a failure of the validator. | Field | Value | |---|---| diff --git a/docs/errors/pipe-load-refusal-error.md b/docs/errors/pipe-load-refusal-error.md new file mode 100644 index 0000000000..9cf0eb8672 --- /dev/null +++ b/docs/errors/pipe-load-refusal-error.md @@ -0,0 +1,21 @@ +--- +title: "Pipe load refusal" +description: "Reference for the `PipeLoadRefusalError` Pipelex error class." +--- + + + +# Pipe load refusal + +Raised by the library load when building one pipe of a bundle raises a refusal of the caller's input that carries no locator of its own. It names the pipe and its file, and bundle validation reports it as an invalid verdict. + +| Field | Value | +|---|---| +| `error_type` | `PipeLoadRefusalError` | +| `title` | Pipe load refusal | +| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-load-refusal-error/` | +| `error_domain` | `input` | +| Defined in | `pipelex.core.pipes.exceptions` | +| Parent class | [`PipelexError`](pipelex-error.md) | + +[Back to Error Reference](index.md) diff --git a/docs/errors/pipe-operator-model-choice-error.md b/docs/errors/pipe-operator-model-choice-error.md index 1c99cc1c24..a64b87cf13 100644 --- a/docs/errors/pipe-operator-model-choice-error.md +++ b/docs/errors/pipe-operator-model-choice-error.md @@ -7,12 +7,14 @@ description: "Reference for the `PipeOperatorModelChoiceError` Pipelex error cla # Pipe operator model choice +Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``) when it is built from its blueprint and a model field names a model the model deck does not define. Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``, and a run refuses the bundle with it before any pipe runs. + | Field | Value | |---|---| | `error_type` | `PipeOperatorModelChoiceError` | | `title` | Pipe operator model choice | | `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-operator-model-choice-error/` | -| `error_domain` | _(inherited from parent)_ | +| `error_domain` | `input` | | Defined in | `pipelex.core.pipes.exceptions` | | Parent class | [`PipelexError`](pipelex-error.md) | diff --git a/docs/errors/validate-bundle-error.md b/docs/errors/validate-bundle-error.md index cf8de68eba..0cf331c815 100644 --- a/docs/errors/validate-bundle-error.md +++ b/docs/errors/validate-bundle-error.md @@ -7,7 +7,7 @@ description: "Reference for the `ValidateBundleError` Pipelex error class." # Validate bundle -Raised when bundle validation fails. +Raised when a bundle is refused while it is loaded or validated: the invalid verdict, carrying one structured item per refusal in ``validation_errors``. Every refusal of the bundle itself becomes one — the parser, factory and pipe-validation errors, a failing dry run, an unknown model (``unknown_model``) and any other refusal of the caller's input — while a failure of the tool or its environment propagates as a no-verdict fault instead. | Field | Value | |---|---| diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.md index 89d51ded58..0fa8a48105 100644 --- a/docs/tools/cli/agent-cli.md +++ b/docs/tools/cli/agent-cli.md @@ -79,6 +79,9 @@ For `bundle`, additional options are available: On a successful run, the envelope also carries `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), each namespaced by `pipe_ref` (`domain.code`). In JSON it is a `pending_signatures` array, in markdown a "Pending signatures" section. A top-down build reads it to see exactly which headers remain to implement. The envelope also carries a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown states the runnability verdict in plain English — runnable when complete, NOT yet runnable above the "Pending signatures" section otherwise. `validate bundle`, `validate method`, and `validate pipe --all` carry them and gate on them: without `--allow-signatures`, the command exits non-zero when `is_runnable` is false. Bare `validate pipe ` omits them, and a `--pipe` slice surfaces them for information without gating. +!!! note "Every refusal of the bundle is an invalid verdict" + A refusal raised while the bundle is loaded or validated is answered with the invalid-verdict envelope — `is_valid: false` and a `validation_errors` array — and exit code `1`, never the no-verdict envelope and exit code `2`, which is kept for failures of the tool or its environment. A pipe naming a model the model deck does not define gives one `pipe_validation` item with `error_type: unknown_model`, carrying `pipe_code`, `domain_code`, `source`, `field_path` (`pipe..model`), `model_reference` (the reference as written), `model_type` and `suggestions` (the deck's close matches), plus a `rename-model` `suggested_fix` when there is exactly one suggestion. Any other refusal of your input with no code of its own gives one item without an `error_type`, located on its pipe when it was raised while that pipe was built. See [Error Model](../../under-the-hood/error-model.md#validation_errors-structured-bundle-validation-diagnostics). + !!! note "Advisory warnings on validate" Whole-bundle and whole-library validate surfaces (`validate bundle`, `validate method`, `validate pipe --all`) also carry a `warnings` array — advisory optionality lints on a VALID bundle that never flip the verdict or the exit code. Each entry has the **same shape as a validation error item** (`category`, `error_type`, `pipe_code`, `domain_code`, `variable_names`, `message`) — this is a different shape from the `init`/`doctor` setup `warnings` (`{type, message}`) documented under Output Contract below. Three families ride the array, always in this order: the useless-`!` lint (`optional_force_redundant`), a `!` (force) input whose slot is guaranteed present in every analyzed flow, so the assertion can never fire; the vacuous-presence lint (`input_presence_vacuous`), an entry-pipe input that must be supplied but whose concept declares no required field, so the empty object satisfies it and a caller cannot tell what to fill in (see [Understanding Optionality](../../building-methods/pipes/understanding-optionality.md)); and the [intent-hint](../../building-methods/concepts/intent-hints.md) lints (`hint_unknown_key`, `hint_unknown_intent`, `hint_inapplicable_intent`). Every whole-bundle validate channel — this CLI, the bare CLI, the builder ops and the protocol validation report — assembles them from one composition point, so which advisories you see does not depend on which command you typed. Hint findings are bounded per site: a site naming many undefined keys reports the first few and then how many more there were, and a long authored key or value is elided in the message. In markdown, warnings render as a "Warnings" section. The array is empty when there is nothing to report; `validate pipe` omits it (no flow context to lint in). `validate bundle`/`validate method` with `--pipe` keep it, and it stays bundle-wide there — the slice narrows the dry run, not the validation. diff --git a/docs/tools/cli/validate.md b/docs/tools/cli/validate.md index ab68a21e76..51d23d6040 100644 --- a/docs/tools/cli/validate.md +++ b/docs/tools/cli/validate.md @@ -122,6 +122,28 @@ pipelex validate method invoice_extractor pipelex validate method invoice_extractor --pipe extract_amounts ``` +## An Invalid Bundle Is a Verdict + +Every refusal of your method raised while it is loaded or validated is reported as an invalid bundle: the grouped `❌ Bundle validation failed` output naming each error's pipe, domain, field and file, with exit code `1`. Exit code `2` is kept for the cases where no verdict could be produced — bad arguments, a target that does not resolve, or a failure of Pipelex or its environment. This holds on every subcommand: `validate pipe` and `validate --all` render a pipe whose dry run fails the same way `validate bundle` does, where they used to print a traceback. + +A pipe that names a model your model deck does not define is the most common case. It is reported as an `Unknown Model` error on that pipe, with the path of the field that names it (`pipe..model`, or `pipe..model_to_structure` for the model a `PipeLLM` structures its output with) and the deck's close matches: + +```text +Pipe Validation Errors: + +1. Unknown Model + Pipe: write_tide_note + Domain: tide_tables + Field: model + → Pipe 'write_tide_note' (PipeLLM), field 'model': Alias 'best-sonet' was not found in the model deck + +Did you mean: @best-gpt + 💡 Suggested fix: Replace model '@best-sonet' of pipe 'write_tide_note' with '@best-gpt', its one close match in the model deck + └─ Path: pipe.write_tide_note.model +``` + +When the deck offers exactly one close match, as here, the error carries a suggested fix that `pipelex fix bundle` applies. With several, choosing among them is yours; `pipelex-agent check-model -t ` and `pipelex-agent models -t ` list what the deck defines. + ## Suggested Fixes When a validation error has a deterministic safe fix, the error output includes a `💡 Suggested fix:` line describing the change, and the report ends with the exact command to apply every suggested fix automatically: @@ -148,6 +170,7 @@ All validation commands check: - Syntax correctness of `.mthds` files - Concept and pipe definitions are valid +- Every model a pipe names is defined in your model deck - Input/output connections are correct - All referenced pipes and concepts exist - Dry-run execution succeeds without errors, which implies the logic is correct and the pipe can be run diff --git a/docs/under-the-hood/error-model.md b/docs/under-the-hood/error-model.md index b600fe3e57..3a0490349e 100644 --- a/docs/under-the-hood/error-model.md +++ b/docs/under-the-hood/error-model.md @@ -83,7 +83,11 @@ A bundle-validation failure (`ValidateBundleError`) aggregates per-error data ac Together the two residuals make the **structured-info invariant total**: every invalid verdict carries a non-empty `validation_errors[]`, never a bare message. The builder tries the channels in order — categorized data, then the `dry_run` residual (the more specific channel), then the `blueprint_validation` fallback — and emits exactly one residual only when no earlier channel produced an item. -Besides `category` and `message`, each item carries whatever identity fields its stage produced — `error_type`, `pipe_code`, `concept_code`, `domain_code`, `field_path`, `field_name`, `variable_names`, `missing_concept_code`, `declared_concepts`, and a `source` (the declaring file path, or the per-content source the in-memory load path was given) that hands a consumer the owning file for cross-file diagnostic placement. When the error has a deterministic remedy, the item also carries a [`suggested_fix`](#suggested_fix-structured-deterministic-fixes). +Besides `category` and `message`, each item carries whatever identity fields its stage produced — `error_type`, `pipe_code`, `concept_code`, `domain_code`, `field_path`, `field_name`, `variable_names`, `missing_concept_code`, `declared_concepts`, the unknown-model locators `model_reference`, `model_type` and `suggestions`, and a `source` (the declaring file path, or the per-content source the in-memory load path was given) that hands a consumer the owning file for cross-file diagnostic placement. When the error has a deterministic remedy, the item also carries a [`suggested_fix`](#suggested_fix-structured-deterministic-fixes). + +**Every refusal of the bundle is a verdict.** A validator answers either a verdict — valid, or invalid with located items — or *no verdict could be produced*, which is reserved for a failure of the tool or its environment. So a refusal raised while loading or validating a bundle becomes an item, never a no-verdict fault. After its class-specific arms, the shared cascade (`translate_to_validate_bundle_error` in `pipelex/pipeline/validate_bundle.py`) turns any other `PipelexError` whose report is `input`-domained into a verdict with one item: `pipe_validation`, with the `pipe_code`, `domain_code` and `source`, when the library load located it on the pipe it was building, and `blueprint_validation` otherwise. That item carries no `error_type`, since a refusal with a closed code has an arm of its own. It keeps the refusal's message only when the refusal authored that message as caller-facing copy, and otherwise names the refusal's title, because the verdict as a whole is kept verbatim under STRICT disclosure. The location comes from the load loop in `LibraryManager.load_from_crate`, the one place that holds a pipe's code, domain and file when building it fails: it raises such a refusal again as a `PipeLoadRefusalError` naming the pipe and the file, `from` the original. A `config` or `runtime` fault, an unclassified one, a `SecurityError`, a `PipeNotFoundError` (which has its own not-found handler) and anything that is not a `PipelexError` still propagate as no verdict. + +**The unknown model is the worked example.** A pipe whose model field names a handle, an alias, a preset or a waterfall the model deck does not define is refused when the pipe is built: the operator's deck check raises `ModelChoiceNotFoundError`, and the operator raises it again as a `PipeOperatorModelChoiceError` located on the pipe and on the field, to which the load adds the file. Every pipe type that names a model (`PipeLLM`, `PipeStructure`, `PipeImgGen`, `PipeExtract`, `PipeSearch`) does this through the same `PipeOperator.locating_model_choice` wrapper, so each gives the same item: `category: pipe_validation`, `error_type: unknown_model`, the `pipe_code`, `domain_code` and `source`, the `field_name` and a `field_path` of `pipe..model` (`pipe..model_to_structure` for a `PipeLLM`'s structuring model), the `model_reference` exactly as written, the `model_type` (`llm`, `text_extractor`, `img_gen` or `search`), and the deck's `suggestions` of the same kind, each spelled as a reference the field accepts. The suggestions stay in the item's `message` too, for consumers that keep only the message. When the deck offers exactly one suggestion, the item carries a safe `rename-model` fix, a `remap_value` of the field from the reference as written to that suggestion. An inline setting table (`model = { model = "…", temperature = 0.2 }`) is not looked up in the deck, so it is not refused here. On a run the same `PipeOperatorModelChoiceError` refuses the bundle before any pipe runs: it is `input`-domained and caller-facing, so the hosted answer is a 422 whose message names the pipe and keeps the deck check's sentence. **Signatures are never an error.** An unimplemented `PipeSignature` reached during validation is a *runnability fact*, not a validation failure: the validator no longer raises on it. The assembled library's outstanding signatures ride the validation report's `pending_signatures`, and `is_runnable = not pending_signatures`. `allow_signatures` is a sweep-mechanics flag only (whether signature pipes are mock-run and listed in `validated_pipes`) — it does not change the verdict, so strict ≡ lenient in the report body. The "is this a failure?" decision moves to the consumer: the CLI exits non-zero on `not is_runnable` unless `--allow-signatures`; the HTTP caller reads `is_runnable`. (The **execute/run** path is different: running a stub still raises `PipeSignatureNotExecutableError`.) @@ -437,7 +441,7 @@ The `validate` surface — both the bare `pipelex validate {bundle,method,pipe}` **The verdict lives in the structured `is_valid` field, not the exit code.** The exit code is a convenience signal for naive shell/CI/Makefile use (`set -e`, `cmd && next`, `if cmd; then`); machine consumers (hooks, the Codex hook, runners) MUST read `is_valid` (and `error_domain`) from the JSON for their block/warn decisions rather than branching on the exit code. Decoupling the verdict from the exit code is what keeps any future exit-code change non-breaking. The 1-vs-2 split is also additive for flat consumers: both stay non-zero, so anything that only tests zero-vs-non-zero is unaffected. -Implementation: the agent CLI threads `exit_code` through `agent_error(...)` (`agent_output.py`, default 1); the validate commands pass `exit_code=2` at every no-verdict site and keep the default 1 on the `ValidateBundleError` arm and the signature gate. The bare CLI sets the code directly via `typer.Exit(...)` in `cli/commands/validate/*` and via the `exit_code` parameter on `handle_model_choice_error` / `handle_model_availability_error` in `cli/error_handlers.py`. Shared boot handlers (`make_pipelex_for_cli`'s gateway/inference/telemetry/model-deck-preset paths) stay exit 1 — they are shared across `run`/`build`/`validate` and out of the validate-policy scope. +Implementation: the agent CLI threads `exit_code` through `agent_error(...)` (`agent_output.py`, default 1); the validate commands pass `exit_code=2` at every no-verdict site and keep the default 1 on the `ValidateBundleError` arm and the signature gate. The bare CLI sets the code directly via `typer.Exit(...)` in `cli/commands/validate/*` and via the `exit_code` parameter on `handle_model_availability_error` in `cli/error_handlers.py`. Every refusal of the bundle is a negative verdict on both CLIs: `validate bundle` and `validate method` validate through the shared cascade, and `validate pipe` and `validate --all` load their libraries through it too, so an unknown model or any other load-time refusal exits 1 with the invalid-bundle output rather than a traceback or the no-verdict exit 2. The bare `validate pipe` and `validate --all` also run their dry run through it, so a pipe whose dry run fails is rendered like any other invalid bundle; the agent CLI answers that failure with its `DryRunError` envelope, `is_valid: false` and exit 1. Shared boot handlers (`make_pipelex_for_cli`'s gateway/inference/telemetry/model-deck-preset paths) stay exit 1 — they are shared across `run`/`build`/`validate` and out of the validate-policy scope. ### API diff --git a/pipelex/base_exceptions.py b/pipelex/base_exceptions.py index af3b714d58..cff3118812 100644 --- a/pipelex/base_exceptions.py +++ b/pipelex/base_exceptions.py @@ -331,6 +331,13 @@ class ValidationErrorItem(BaseModel): missing_concept_code: str | None = None missing_pipe_code: str | None = None declared_concepts: list[str] | None = None + # The unknown-model locators, carried by an ``unknown_model`` item: the model reference exactly as + # the author wrote it, the model type the field takes (``llm``, ``text_extractor``, ``img_gen``, + # ``search``), and the deck's close matches of the same kind, each spelled as a reference the field + # accepts. Optional and additive, like ``declared_concepts``: every other item serializes unchanged. + model_reference: str | None = None + model_type: str | None = None + suggestions: list[str] | None = None # Structured, deterministic fix for this error, when the fix planner derived one from the # enriched error data. Optional and additive: non-fixable items serialize unchanged under # ``exclude_none``. diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.md index 882d93a14a..8388114e8f 100644 --- a/pipelex/cli/agent_cli/CLAUDE.md +++ b/pipelex/cli/agent_cli/CLAUDE.md @@ -107,5 +107,6 @@ commands/ - **Init**: All commands that need Pipelex use `make_pipelex_for_agent_cli(library_dirs)`. It catches init errors and routes them through `agent_error()`. - **Async core**: Run and validate are async — commands use `asyncio.run()`. - **File convention**: Generated outputs go to `mthds-wip/` with incremental naming (`pipeline_01/`, `pipeline_02/`). Exception: `run method` on a fetched target (a method address or GitHub URL) anchors its run outputs in `results/` under the caller's CWD via `run_pipeline_core`'s `output_dir_override` — the fetched clone is an ephemeral temp dir deleted at process exit, so bundle-adjacent outputs would be lost. +- **Validate refusals are verdicts**: the `validate` commands run their library load inside `translate_to_validate_bundle_error` (`pipelex/pipeline/validate_bundle.py`), so every input-domained refusal raised while loading a bundle, the unknown model (`PipeOperatorModelChoiceError`, an `unknown_model` item) included, reaches the command as a `ValidateBundleError` and leaves with the invalid-verdict envelope and exit 1. No `validate` command keeps an arm of its own for a model-choice error; exit 2 stays for a failure of the tool or its environment (config, runtime, security). - **Method-reference failures**: the `run`/`validate`/`inputs` method commands call `resolve_method_target(..., raise_ref_errors=True)` and shape the typed `MethodRefError` subclasses into `agent_error(...)` themselves (validate uses `exit_code=2`, its no-verdict convention) — the human CLI's red-text rendering stays in `resolve_method_target`'s default arm. - **TOML handling**: Uses `tomlkit` (not `tomllib`) to preserve formatting and inline tables. diff --git a/pipelex/cli/agent_cli/commands/agent_output.py b/pipelex/cli/agent_cli/commands/agent_output.py index 3dc0283ef1..27f8c90f6a 100644 --- a/pipelex/cli/agent_cli/commands/agent_output.py +++ b/pipelex/cli/agent_cli/commands/agent_output.py @@ -104,13 +104,17 @@ def get_agent_cli_error_format() -> CliOutputFormat: # drift test — a dead entry is exactly the rot it exists to prevent. When the # derived domain is not the one this CLI wants, fix it on the class (declare an # explicit error_domain, as ModelChoiceNotFoundError does), never here. +# An unknown model is the author's input fault, whether it is raised bare by the deck check or located +# on its pipe: the next step is to correct the reference, not to diagnose the installation. +_UNKNOWN_MODEL_HINT = ( + "Check model name for typos. Use 'pipelex-agent check-model -t ' " + "to validate or 'pipelex-agent models -t ' to list available models." +) + AGENT_ERROR_HINTS: dict[str, str] = { # Model/routing errors - "ModelChoiceNotFoundError": ( - "Check model name for typos. Use 'pipelex-agent check-model -t ' " - "to validate or 'pipelex-agent models -t ' to list available models." - ), - "PipeOperatorModelChoiceError": "Run 'pipelex-agent doctor' to check available models and routing configuration", + "ModelChoiceNotFoundError": _UNKNOWN_MODEL_HINT, + "PipeOperatorModelChoiceError": _UNKNOWN_MODEL_HINT, "PipeOperatorModelAvailabilityError": "Run 'pipelex-agent doctor' to check available models and verify API keys", "ModelDeckPresetValidatonError": ( "Run 'pipelex-agent doctor' to check model configuration. " @@ -210,7 +214,6 @@ def get_agent_cli_error_format() -> CliOutputFormat: "CodegenLockError": "input", # config = environment/config changes needed "ClientAuthenticationError": "config", - "PipeOperatorModelChoiceError": "config", "PipeOperatorModelAvailabilityError": "config", "BinaryNotFoundError": "config", "InitConfigError": "config", diff --git a/pipelex/cli/agent_cli/commands/validate/_validate_core.py b/pipelex/cli/agent_cli/commands/validate/_validate_core.py index df07de7f20..de1e56549d 100644 --- a/pipelex/cli/agent_cli/commands/validate/_validate_core.py +++ b/pipelex/cli/agent_cli/commands/validate/_validate_core.py @@ -17,7 +17,7 @@ from pipelex.pipeline.blueprint_selection import collect_entry_pipe_refs from pipelex.pipeline.bundle_validator import BundleValidator from pipelex.pipeline.execution_seams import acquire_library -from pipelex.pipeline.validate_bundle import build_pending_signatures, build_validated_pipes, validate_bundle +from pipelex.pipeline.validate_bundle import build_pending_signatures, build_validated_pipes, translate_to_validate_bundle_error, validate_bundle if TYPE_CHECKING: from pathlib import Path @@ -42,10 +42,14 @@ async def validate_all_core(*, library_dirs: list[Path] | None = None, allow_sig # signatures and must read the LIBRARY-WIDE pending set BEFORE teardown — acquire_and_validate # returns only the per-pipe status map and tears the library down before we could compute it. prev_library_id = get_current_library_id_or_none() - acquired_id, _ = acquire_library( - library_id="", - library_dirs=[str(library_dir) for library_dir in library_dirs] if library_dirs else None, - ) + # A refusal of the libraries while loading them is the invalid verdict, through the shared + # bundle-loading cascade, as on `validate bundle`. acquire_library tears its library down itself on + # a failed load. + with translate_to_validate_bundle_error(): + acquired_id, _ = acquire_library( + library_id="", + library_dirs=[str(library_dir) for library_dir in library_dirs] if library_dirs else None, + ) try: # acquire_library left the freshly-acquired library current, so the inner sweep targets it # (it filters signatures in strict mode itself). The returned map is keyed by namespaced pipe_ref. @@ -156,7 +160,10 @@ async def validate_pipe_core( effective_dirs, _ = resolve_library_dirs(library_dirs) if effective_dirs: - library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs) + # A refusal of the libraries while loading them is the invalid verdict, through the shared + # bundle-loading cascade, as on `validate bundle`. + with translate_to_validate_bundle_error(): + library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs) the_pipe = get_required_entry_pipe(pipe_code=pipe_code) dry_run_results = await BundleValidator().validate_pipes(pipes=[the_pipe], library_id=library_id, allow_signatures=allow_signatures) diff --git a/pipelex/cli/agent_cli/commands/validate/bundle_cmd.py b/pipelex/cli/agent_cli/commands/validate/bundle_cmd.py index 27cc2c0b0a..fb4aab0b8e 100644 --- a/pipelex/cli/agent_cli/commands/validate/bundle_cmd.py +++ b/pipelex/cli/agent_cli/commands/validate/bundle_cmd.py @@ -19,7 +19,6 @@ validate_bundle_core, validate_pipe_in_bundle_core, ) -from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError from pipelex.graph.graph_rendering import GraphFormat from pipelex.libraries.pipe.exceptions import PipeNotFoundError from pipelex.mthds_parsing.exceptions import MthdsParserError @@ -200,17 +199,6 @@ def validate_bundle_cmd( # footer. Signatures never reach here (they are a runnability fact, gated above). agent_error_validate_bundle(exc, bundle_path=Path(bundle_path), library_dirs=library_dirs, allow_signatures=allow_signatures) - except PipeOperatorModelChoiceError as exc: - agent_error( - exc.message, - error_type="PipeOperatorModelChoiceError", - cause=exc, - exit_code=2, - pipe_code=exc.pipe_code, - model_type=str(exc.model_type), - model_choice=str(exc.model_choice), - ) - except PipeOperatorModelAvailabilityError as exc: availability_extra: dict[str, Any] = { "pipe_code": exc.pipe_code, diff --git a/pipelex/cli/agent_cli/commands/validate/method_cmd.py b/pipelex/cli/agent_cli/commands/validate/method_cmd.py index 794914167c..cf841e73d6 100644 --- a/pipelex/cli/agent_cli/commands/validate/method_cmd.py +++ b/pipelex/cli/agent_cli/commands/validate/method_cmd.py @@ -21,7 +21,6 @@ validate_pipe_in_bundle_core, ) from pipelex.cli.method_resolver import resolve_method_target -from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError from pipelex.methods.exceptions import MethodRefError from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError from pipelex.pipelex import Pipelex @@ -129,17 +128,6 @@ def validate_method_cmd( # structured envelope; markdown renders the items as prose with a fix-aware footer. agent_error_validate_bundle(exc, bundle_path=bundle_path, library_dirs=library_dirs_paths, allow_signatures=allow_signatures) - except PipeOperatorModelChoiceError as exc: - agent_error( - exc.message, - error_type="PipeOperatorModelChoiceError", - cause=exc, - exit_code=2, - pipe_code=exc.pipe_code, - model_type=str(exc.model_type), - model_choice=str(exc.model_choice), - ) - except PipeOperatorModelAvailabilityError as exc: availability_extra: dict[str, Any] = { "pipe_code": exc.pipe_code, diff --git a/pipelex/cli/agent_cli/commands/validate/pipe_cmd.py b/pipelex/cli/agent_cli/commands/validate/pipe_cmd.py index 2f7dce6a1d..36f660819d 100644 --- a/pipelex/cli/agent_cli/commands/validate/pipe_cmd.py +++ b/pipelex/cli/agent_cli/commands/validate/pipe_cmd.py @@ -19,7 +19,6 @@ validate_pipe_core, ) from pipelex.cli.method_resolver import resolve_pipe_from_exports -from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError from pipelex.libraries.pipe.exceptions import PipeNotFoundError from pipelex.mthds_parsing.helpers import MTHDS_EXTENSION, is_pipelex_file from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError @@ -104,17 +103,6 @@ def validate_pipe_cmd( validation_errors=extract_validation_errors(exc), ) - except PipeOperatorModelChoiceError as exc: - agent_error( - exc.message, - error_type="PipeOperatorModelChoiceError", - cause=exc, - pipe_code=exc.pipe_code, - model_type=str(exc.model_type), - model_choice=str(exc.model_choice), - exit_code=2, - ) - except PipeOperatorModelAvailabilityError as exc: agent_error( str(exc), @@ -204,17 +192,6 @@ def validate_pipe_cmd( validation_errors=extract_validation_errors(exc), ) - except PipeOperatorModelChoiceError as exc: - agent_error( - exc.message, - error_type="PipeOperatorModelChoiceError", - cause=exc, - pipe_code=exc.pipe_code, - model_type=str(exc.model_type), - model_choice=str(exc.model_choice), - exit_code=2, - ) - except PipeOperatorModelAvailabilityError as exc: availability_extra: dict[str, Any] = { "pipe_code": exc.pipe_code, diff --git a/pipelex/cli/commands/validate/_validate_core.py b/pipelex/cli/commands/validate/_validate_core.py index 3294c9c716..ef7e10657b 100644 --- a/pipelex/cli/commands/validate/_validate_core.py +++ b/pipelex/cli/commands/validate/_validate_core.py @@ -11,10 +11,8 @@ from pipelex.cli.error_handlers import ( ErrorContext, handle_model_availability_error, - handle_model_choice_error, handle_validate_bundle_error, ) -from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError from pipelex.interpreter_hub import ( get_library_manager, get_pipe_library, @@ -32,7 +30,7 @@ from pipelex.pipeline.bundle_validator import BundleValidator from pipelex.pipeline.exceptions import ValidateBundleError from pipelex.pipeline.execution_seams import load_libraries_and_activate -from pipelex.pipeline.validate_bundle import build_pending_signatures, validate_bundle +from pipelex.pipeline.validate_bundle import build_pending_signatures, translate_to_validate_bundle_error, validate_bundle from pipelex.runtime_hub import get_console, get_telemetry_manager from pipelex.system.runtime import IntegrationMode from pipelex.system.telemetry.events import EventProperty @@ -80,7 +78,10 @@ def do_validate_all_libraries_and_dry_run(*, library_dirs: list[Path] | None = N # Single public composer for the open/set/load ceremony — leaves the library loaded and # current for the sweep below, owning the standard 3-tier dir resolution and load-failure # teardown. No teardown on success here: the caller (validate_pipe_cmd) owns Pipelex teardown. - load_libraries_and_activate(library_dirs) + # The shared bundle-loading cascade turns a refusal of the libraries into the invalid + # verdict (exit 1) rendered below, exactly as `validate bundle` does. + with translate_to_validate_bundle_error(): + load_libraries_and_activate(library_dirs) # The pipe list is needed only to render the "Validating N" line; validate_current_library # re-derives its own sweep candidates from the current library (validate_pipes excludes @@ -92,8 +93,10 @@ def do_validate_all_libraries_and_dry_run(*, library_dirs: list[Path] | None = N typer.echo(f"Validating {count_with_noun(count=len(pipes), singular='pipe')} from: {dirs_str}") # validate_current_library owns the static wiring pass and the single PIPE_DRY_RUN telemetry - # event — sweeping the library we just loaded, without teardown. - asyncio.run(BundleValidator().validate_current_library(allow_signatures=allow_signatures)) + # event — sweeping the library we just loaded, without teardown. A pipe whose dry run fails + # raises DryRunError, which the cascade turns into the same invalid verdict. + with translate_to_validate_bundle_error(): + asyncio.run(BundleValidator().validate_current_library(allow_signatures=allow_signatures)) # Advisory lints over the whole loaded library (the optionality lint's cross-flow # aggregation needs every flow) — printed even when the signature gate below exits @@ -120,10 +123,12 @@ def do_validate_all_libraries_and_dry_run(*, library_dirs: list[Path] | None = N typer.echo( f"Setup sequence passed OK, config and pipelines are validated.{_format_signatures_summary_suffix(signature_count=signature_count)}" ) + except ValidateBundleError as bundle_error: + # A produced negative verdict (exit 1): a refusal of the libraries while loading them, an unknown + # model among them, or a pipe whose dry run fails — rendered like any other invalid bundle. + handle_validate_bundle_error(bundle_error, library_dirs=library_dirs, allow_signatures=allow_signatures) except PipeOperatorModelAvailabilityError as exc: handle_model_availability_error(exc, context=ErrorContext.VALIDATION, exit_code=2) - except PipeOperatorModelChoiceError as exc: - handle_model_choice_error(exc, context=ErrorContext.VALIDATION, exit_code=2) async def _validate_pipe_or_bundle( @@ -177,19 +182,28 @@ async def _validate_pipe_or_bundle( set_current_library(library_id=library_id) effective_dirs, _ = resolve_library_dirs(library_dirs) - if effective_dirs: - library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs) + try: + # The load and the dry run go through the shared bundle-loading cascade, so a refusal of the + # libraries and a pipe whose dry run fails are invalid verdicts (exit 1) rendered like any + # other invalid bundle. The entry-pipe lookup between them stays outside it: a code that names + # no pipe, or several, is an unresolvable target — no verdict, handled by execute_validate. + if effective_dirs: + with translate_to_validate_bundle_error(): + library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs) - pipe = get_required_entry_pipe(pipe_code=pipe_code) - typer.echo(f"Validating pipe '{pipe_code}'...") - # Signatures are never an error (D-B): a single-pipe validation reaching a PipeSignature - # dry-runs trivially (the placeholder mints a mock). validate pipe makes no library-wide - # runnability claim — pending_signatures is a bundle-surface fact — so there is no gate here. - await BundleValidator().validate_pipes( - pipes=[pipe], - library_id=library_id, - allow_signatures=allow_signatures, - ) + pipe = get_required_entry_pipe(pipe_code=pipe_code) + typer.echo(f"Validating pipe '{pipe_code}'...") + # Signatures are never an error (D-B): a single-pipe validation reaching a PipeSignature + # dry-runs trivially (the placeholder mints a mock). validate pipe makes no library-wide + # runnability claim — pending_signatures is a bundle-surface fact — so there is no gate here. + with translate_to_validate_bundle_error(): + await BundleValidator().validate_pipes( + pipes=[pipe], + library_id=library_id, + allow_signatures=allow_signatures, + ) + except ValidateBundleError as bundle_error: + handle_validate_bundle_error(bundle_error, library_dirs=library_dirs, allow_signatures=allow_signatures) signature_count = len(collect_signature_refs(pipe=pipe)) typer.secho( f"Successfully validated pipe '{pipe_code}'{_format_signatures_summary_suffix(signature_count=signature_count)}", @@ -258,8 +272,6 @@ def execute_validate( err=True, ) raise typer.Exit(2) from exc - except PipeOperatorModelChoiceError as exc: - handle_model_choice_error(exc, context=ErrorContext.VALIDATION, exit_code=2) except PipeOperatorModelAvailabilityError as exc: handle_model_availability_error(exc, context=ErrorContext.VALIDATION, exit_code=2) finally: diff --git a/pipelex/cli/error_handlers.py b/pipelex/cli/error_handlers.py index 41f33d7eae..b0fb181363 100644 --- a/pipelex/cli/error_handlers.py +++ b/pipelex/cli/error_handlers.py @@ -124,8 +124,9 @@ def handle_model_choice_error(exc: PipeOperatorModelChoiceError, *, context: Err Args: exc: The model choice error exception context: Context for the error message - exit_code: Process exit code. The validate surface passes 2 (a no-verdict - setup/config error per its 0/1/2 policy); other contexts keep the default 1. + exit_code: Process exit code; the default is 1. The validate surface never reaches + this handler: an unknown model is an invalid verdict there, rendered by + :func:`handle_validate_bundle_error`. """ console = get_console() print_traceback_if_requested(console=console) @@ -157,8 +158,9 @@ def handle_model_availability_error(exc: PipeOperatorModelAvailabilityError, *, Args: exc: The model availability error exception context: Context for the error message - exit_code: Process exit code. The validate surface passes 2 (a no-verdict - setup/config error per its 0/1/2 policy); other contexts keep the default 1. + exit_code: Process exit code; the default is 1. The validate surface never reaches + this handler: an unknown model is an invalid verdict there, rendered by + :func:`handle_validate_bundle_error`. """ console = get_console() print_traceback_if_requested(console=console) diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.py index 9505560d03..468e2ec443 100644 --- a/pipelex/cogt/exceptions.py +++ b/pipelex/cogt/exceptions.py @@ -195,7 +195,11 @@ class SdkTypeError(CogtError): class ModelChoiceNotFoundError(CogtError): - """Error raised when a model choice cannot be found in the model deck. + """Raised when a model reference names a handle, alias, preset or waterfall the model deck does not define: + by the deck check a pipe runs when it is built, and by the deck when a run resolves a reference. When a + pipe is built, the pipe operator raises it again as a ``PipeOperatorModelChoiceError`` located on the pipe + and the field, so a bundle naming an unknown model is an invalid validation verdict (error type + ``unknown_model``), never a failure of the validator. Includes available options and migration hints in error message. """ diff --git a/pipelex/core/exceptions.py b/pipelex/core/exceptions.py index 2743f87035..a485b38153 100644 --- a/pipelex/core/exceptions.py +++ b/pipelex/core/exceptions.py @@ -77,8 +77,12 @@ class PipesAndConceptValidationErrorData(BaseModel): field_name: str | None = Field(default=None, description="Specific field that failed") # === Error Classification === - error_type: PipeValidationErrorType = Field( - description="Type of pipe/concept validation error", + # ``None`` for a refusal no closed code fits: a load-time refusal located on a pipe but carrying no + # code of its own (the general arm of the bundle-loading cascade), exactly as the parse-level + # residual carries none rather than a code that would claim to know which fault occurred. + error_type: PipeValidationErrorType | None = Field( + default=None, + description="Type of pipe/concept validation error, or None when no closed code fits the refusal", ) # === Error Details === @@ -105,3 +109,11 @@ class PipesAndConceptValidationErrorData(BaseModel): description="The pipe's currently declared inputs mapping, rendered like expected_inputs, " "so a fix planner can diff the two without file access", ) + + # === Unknown-model locators (for unknown_model errors) === + model_reference: str | None = Field(default=None, description="The model reference exactly as the author wrote it") + model_type: str | None = Field(default=None, description="The model type the field takes (llm, text_extractor, img_gen, search)") + suggestions: list[str] | None = Field( + default=None, + description="The deck's close matches of the same kind, each spelled as a reference the field accepts", + ) diff --git a/pipelex/core/pipes/exceptions.py b/pipelex/core/pipes/exceptions.py index 77121f70bf..f324390956 100644 --- a/pipelex/core/pipes/exceptions.py +++ b/pipelex/core/pipes/exceptions.py @@ -1,11 +1,6 @@ -from typing_extensions import override - -from pipelex.base_exceptions import PipelexError -from pipelex.cogt.extract.extract_setting import ExtractModelChoice -from pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice -from pipelex.cogt.llm.llm_setting import LLMModelChoice +from pipelex.base_exceptions import ErrorDomain, PipelexError +from pipelex.cogt.exceptions import ModelChoiceNotFoundError from pipelex.cogt.model_backends.model_type import ModelType -from pipelex.cogt.models.model_reference import ModelReference from pipelex.system.pipe_run_mode import PipeRunMode from pipelex.validation_error_types import PipeFactoryErrorType, PipeValidationErrorType @@ -61,41 +56,116 @@ def __init__(self, message: str, run_mode: PipeRunMode, pipe_code: str): class PipeOperatorModelChoiceError(PipelexError): + """Raised by a pipe operator (``PipeLLM``, ``PipeStructure``, ``PipeImgGen``, ``PipeExtract``, ``PipeSearch``) + when it is built from its blueprint and a model field names a model the model deck does not define. + Bundle validation reports it as an invalid verdict whose item has the error type ``unknown_model``, + and a run refuses the bundle with it before any pipe runs. + + The pipe operator raises it from the ``ModelChoiceNotFoundError`` its deck check raised, located on + the pipe (its code, type and domain) and on the field (``model``, or ``model_to_structure`` on a + ``PipeLLM``), and the library load adds the file the pipe is declared in. Every pipe type that names + a model raises this same error, so every one of them gives the same verdict item. + + The message names the pipe and the field, then keeps the deck check's sentence and its suggestions, + so it is caller-facing copy: it names only the caller's own model reference and the deck's public + handles, and it must reach a hosted caller under STRICT disclosure. + """ + + error_domain = ErrorDomain.INPUT + _authors_caller_facing_message = True + def __init__( self, message: str, + *, pipe_type: str, pipe_code: str, + domain_code: str, + field_name: str, model_type: ModelType, - model_choice: LLMModelChoice | ExtractModelChoice | ImgGenModelChoice, + model_choice: str, + suggestions: list[str] | None = None, + source: str | None = None, ): self.pipe_type = pipe_type self.pipe_code = pipe_code + self.domain_code = domain_code + self.field_name = field_name self.model_type = model_type + # The reference exactly as the author wrote it (``gpt-5.1``, ``@best-sonet``, ``$writting-factual``). self.model_choice = model_choice + # The deck's close matches of the same kind, each spelled as a reference the field accepts. + self.suggestions = suggestions or [] + # The file declaring the pipe. The operator does not know it — pipes carry no source — so the + # library load fills it from the crate's source map before the error leaves the load loop. + self.source = source super().__init__(message) - def desc(self) -> str: - msg = f"{self.message}" - msg += f" • pipe='{self.pipe_code}' ({self.pipe_type})" - msg += f" • model_type='{self.model_type}'" - - # Extract the choice identifier from the model_choice union type - if isinstance(self.model_choice, str): - # It's a raw string (shouldn't happen but handle it) - msg += f" • choice='{self.model_choice}'" - elif isinstance(self.model_choice, ModelReference): - # It's a ModelReference with kind and name - msg += f" • choice='{self.model_choice.raw}' ({self.model_choice.kind})" - else: - # It's a Setting object with a model field and optional desc() - msg += f" • choice={self.model_choice.desc()}" + @classmethod + def make_from_model_choice_not_found( + cls, + *, + model_choice_error: ModelChoiceNotFoundError, + pipe_type: str, + pipe_code: str, + domain_code: str, + field_name: str, + ) -> "PipeOperatorModelChoiceError": + """Locate the deck check's refusal on the pipe and the field that named the model.""" + message = f"Pipe '{pipe_code}' ({pipe_type}), field '{field_name}': {model_choice_error.message}" + return cls( + message, + pipe_type=pipe_type, + pipe_code=pipe_code, + domain_code=domain_code, + field_name=field_name, + model_type=model_choice_error.model_type, + model_choice=model_choice_error.model_choice, + suggestions=list(model_choice_error.suggestions), + ) + + +class PipeLoadRefusalError(PipelexError): + """Raised by the library load when building one pipe of a bundle raises a refusal of the caller's input + that carries no locator of its own. It names the pipe and its file, and bundle validation reports it + as an invalid verdict. + + The load raises it ``from`` the refusal, so the original stays the ``__cause__``. Bundle validation + turns it into one ``pipe_validation`` item carrying the pipe code, the domain and the source, with no + ``error_type``, since the refusal has no closed code. Only an ``input``-domained refusal is wrapped: a + fault of the configuration or of the runtime keeps its own identity and stays a no-verdict fault. + + Its message is built only from caller-facing material, the caller's own pipe code and either the + refusal's message, when that message is caller-facing, or the refusal's title, so it is kept under + STRICT disclosure. + """ - return msg + error_domain = ErrorDomain.INPUT + _authors_caller_facing_message = True + + def __init__(self, message: str, *, pipe_code: str, domain_code: str, source: str | None): + self.pipe_code = pipe_code + self.domain_code = domain_code + self.source = source + super().__init__(message) - @override - def __str__(self) -> str: - return self.desc() + @classmethod + def make_from_refusal(cls, *, refusal: PipelexError, pipe_code: str, domain_code: str, source: str | None) -> "PipeLoadRefusalError": + """Locate a refusal on the pipe being built, keeping its message only when it is caller-facing.""" + message = f"Pipe '{pipe_code}' could not be loaded: {caller_facing_refusal_text(refusal=refusal)}" + return cls(message, pipe_code=pipe_code, domain_code=domain_code, source=source) + + +def caller_facing_refusal_text(*, refusal: PipelexError) -> str: + """The refusal's message when it was authored as caller-facing copy, otherwise its title. + + A validation verdict is caller-facing as a whole — ``ValidateBundleError`` keeps its message under + STRICT disclosure — so text taken from a refusal onto a verdict item must be caller-facing too: a + refusal whose message is internal is named by its title, which is always public. + """ + if refusal.to_error_report().caller_facing_message: + return refusal.message + return type(refusal).title() class PipeValidationError(ValueError): diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py index 11ff778020..a43a24f80c 100644 --- a/pipelex/libraries/library_manager.py +++ b/pipelex/libraries/library_manager.py @@ -1,5 +1,6 @@ import uuid -from contextlib import ExitStack +from collections.abc import Generator +from contextlib import ExitStack, contextmanager from graphlib import CycleError, TopologicalSorter from pathlib import Path from typing import TYPE_CHECKING @@ -15,12 +16,14 @@ import pipelex.builder as builder_pkg # package import — used for __file__ path from pipelex import log +from pipelex.base_exceptions import PipelexError, SecurityError, error_domain_is_input from pipelex.config import is_pipe_func_sandbox_hosted from pipelex.core.concepts.concept_blueprint import ConceptBlueprint from pipelex.core.concepts.concept_factory import ConceptFactory from pipelex.core.concepts.native.concept_native import NativeConceptCode from pipelex.core.domains.domain_blueprint import DomainBlueprint from pipelex.core.domains.domain_factory import DomainFactory +from pipelex.core.pipes.exceptions import PipeLoadRefusalError, PipeOperatorModelChoiceError from pipelex.core.qualified_ref import QualifiedRef from pipelex.core.stuffs.structured_content import StructuredContent from pipelex.core.validation import report_validation_error @@ -99,6 +102,38 @@ def _find_methods_dirs_from_blueprints(blueprints: list[PipelexBundleBlueprint]) return result +@contextmanager +def _locating_pipe_build_refusals(*, pipe_code: str, domain_code: str, source: str | None) -> Generator[None, None, None]: + """Let a refusal raised while building one pipe leave the load loop located on that pipe and its file. + + The loop is the one place that holds the pipe's code, its domain and the file it is declared in at + the moment a build fails: pipes carry no source, and the pipe-source map is filled only after a + pipe is built. So the location is attached here, and bundle validation reads it off the refusal: + + - An unknown model is already located on its pipe and field by the operator + (``PipeOperatorModelChoiceError``); the loop adds the file and lets it go on under its own class, + which the run and build surfaces render with their dedicated panel. + - Any other refusal of the caller's input (an ``input``-domained ``PipelexError``) is raised again + as a ``PipeLoadRefusalError`` naming the pipe and the file, ``from`` the original. + - Everything else leaves untouched: a configuration or runtime fault keeps its identity and stays a + no-verdict fault, a security refusal is never absorbed into a verdict, a library error keeps the + structured items its own arm forwards, and pydantic's ``ValidationError`` and the + ``PipeValidationError`` family (not ``PipelexError``s) keep their categorizers. + """ + try: + yield + except PipeOperatorModelChoiceError as model_choice_error: + if model_choice_error.source is None: + model_choice_error.source = source + raise + except (SecurityError, LibraryError): + raise + except PipelexError as refusal: + if not error_domain_is_input(refusal.to_error_report().error_domain): + raise + raise PipeLoadRefusalError.make_from_refusal(refusal=refusal, pipe_code=pipe_code, domain_code=domain_code, source=source) from refusal + + class LibraryManager(LibraryManagerAbstract): def __init__(self): # UNTITLED library is the fallback library for all others @@ -528,16 +563,17 @@ def load_from_crate(self, *, library_id: str, crate: LibraryCrate) -> list[PipeA concept_codes_for_domain = domain_concept_codes.get(domain_code, []) - pipe = PipeFactory[PipeAbstract].make_from_blueprint( - domain_code=domain_code, - pipe_code=pipe_code, - blueprint=pipe_blueprint, - concept_codes_from_the_same_domain=concept_codes_for_domain, - ) + source = crate.source_map.get(pipe_ref) + with _locating_pipe_build_refusals(pipe_code=pipe_code, domain_code=domain_code, source=source): + pipe = PipeFactory[PipeAbstract].make_from_blueprint( + domain_code=domain_code, + pipe_code=pipe_code, + blueprint=pipe_blueprint, + concept_codes_from_the_same_domain=concept_codes_for_domain, + ) all_pipes.append(pipe) # Track source file for this pipe (used by get_pipe_source) - source = crate.source_map.get(pipe_ref) if source: self._pipe_source_maps.setdefault(library_id, {})[pipe_ref] = source diff --git a/pipelex/mthds_parsing/handle_pipe_errors.py b/pipelex/mthds_parsing/handle_pipe_errors.py index 79181d3ae4..f3b389e5b4 100644 --- a/pipelex/mthds_parsing/handle_pipe_errors.py +++ b/pipelex/mthds_parsing/handle_pipe_errors.py @@ -5,7 +5,7 @@ from pydantic_core import ErrorDetails from pipelex.core.exceptions import PipeFactoryErrorData, PipesAndConceptValidationErrorData -from pipelex.core.pipes.exceptions import PipeFactoryError, PipeValidationError +from pipelex.core.pipes.exceptions import PipeFactoryError, PipeOperatorModelChoiceError, PipeValidationError from pipelex.validation_error_types import PipeValidationErrorType @@ -235,3 +235,34 @@ def categorize_pipe_factory_error( declared_concepts=factory_error.declared_concepts, message=factory_error.message, ) + + +def categorize_pipe_operator_model_choice_error( + *, + model_choice_error: PipeOperatorModelChoiceError, +) -> PipesAndConceptValidationErrorData: + """Categorize an unknown model into the ``unknown_model`` error data, located on its pipe and field. + + The ``field_path`` addresses the reference in the bundle (``pipe..model``, or + ``pipe..model_to_structure`` on a ``PipeLLM``). The reference as written, the model type and + the deck's suggestions ride as their own fields, and the suggestions stay in the message too, for + the consumers that keep only an item's message. + + Args: + model_choice_error: The located unknown-model refusal a pipe operator raised when it was built. + + Returns: + PipesAndConceptValidationErrorData with the unknown-model locators populated + """ + return PipesAndConceptValidationErrorData( + error_type=PipeValidationErrorType.UNKNOWN_MODEL, + domain_code=model_choice_error.domain_code, + source=model_choice_error.source, + pipe_code=model_choice_error.pipe_code, + field_name=model_choice_error.field_name, + field_path=f"pipe.{model_choice_error.pipe_code}.{model_choice_error.field_name}", + message=model_choice_error.message, + model_reference=model_choice_error.model_choice, + model_type=model_choice_error.model_type, + suggestions=list(model_choice_error.suggestions), + ) diff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.py index 771c415b0e..2583e3bb14 100644 --- a/pipelex/pipe_operators/extract/pipe_extract.py +++ b/pipelex/pipe_operators/extract/pipe_extract.py @@ -3,7 +3,6 @@ from pydantic import model_validator from typing_extensions import Self, override -from pipelex.cogt.exceptions import ModelChoiceNotFoundError from pipelex.cogt.extract.extract_input import ExtractInput from pipelex.cogt.extract.extract_setting import ExtractModelChoice from pipelex.cogt.models.model_deck_check import check_extract_choice_with_deck @@ -55,11 +54,8 @@ def validate_fields(self) -> Self: @override def validate_inputs_static(self): if self.extract_choice: - try: + with self.locating_model_choice(field_name="model"): check_extract_choice_with_deck(extract_choice=self.extract_choice) - except ModelChoiceNotFoundError as exc: - msg = f"Extract choice '{self.extract_choice}' was not found in the model deck" - raise ValueError(msg) from exc @override def validate_inputs_with_library(self): diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py index 5506162c25..a63669305f 100644 --- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py +++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py @@ -65,7 +65,8 @@ def required_variables(self) -> set[str]: @override def validate_inputs_static(self): if self.img_gen_choice: - check_img_gen_choice_with_deck(img_gen_choice=self.img_gen_choice) + with self.locating_model_choice(field_name="model"): + check_img_gen_choice_with_deck(img_gen_choice=self.img_gen_choice) self._validate_param_support_against_model_rules() # Guard-lint (D7): every reference to a declared-optional input must be guarded. diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py index dd7d729292..017230c383 100644 --- a/pipelex/pipe_operators/llm/pipe_llm.py +++ b/pipelex/pipe_operators/llm/pipe_llm.py @@ -8,6 +8,7 @@ from pipelex.cogt.exceptions import LLMCompletionError from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting, LLMSettingChoices from pipelex.cogt.models.model_deck_check import check_llm_choice_with_deck +from pipelex.cogt.models.model_reference import ModelReference from pipelex.core.concepts.concept_factory import ConceptFactory from pipelex.core.concepts.native.concept_native import NativeConceptCode from pipelex.core.domains.domain import SpecialDomain @@ -62,8 +63,14 @@ class PipeLLM(PipeOperator[PipeLLMOutput]): @override def validate_inputs_static(self): if self.llm_choices: - for llm_choice_ref in self.llm_choices.list_choice_references(): - check_llm_choice_with_deck(llm_choice=llm_choice_ref) + # Checked field by field, in declaration order, so a refusal names the field the author wrote + # the reference in: `for_text` comes from the blueprint's `model`, `for_object` from its + # `model_to_structure`. An inline setting table is not a reference and is not looked up. + for field_name, llm_choice in (("model", self.llm_choices.for_text), ("model_to_structure", self.llm_choices.for_object)): + if not isinstance(llm_choice, ModelReference): + continue + with self.locating_model_choice(field_name=field_name): + check_llm_choice_with_deck(llm_choice=llm_choice) needed_inputs = self.needed_inputs() required_variable_paths = self.required_variables() diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py index e91c956e88..9d37febf9d 100644 --- a/pipelex/pipe_operators/pipe_operator.py +++ b/pipelex/pipe_operators/pipe_operator.py @@ -1,10 +1,13 @@ from abc import abstractmethod +from collections.abc import Generator +from contextlib import contextmanager from typing import TYPE_CHECKING, Generic, Literal, TypeVar, final from typing_extensions import override -from pipelex.cogt.exceptions import ModelNotFoundError, ModelWaterfallError +from pipelex.cogt.exceptions import ModelChoiceNotFoundError, ModelNotFoundError, ModelWaterfallError from pipelex.core.memory.working_memory import WorkingMemory +from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError from pipelex.core.pipes.pipe_output import PipeOutput from pipelex.pipe_machinery.pipe_abstract import PipeAbstract from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError @@ -26,6 +29,28 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]): def class_name(self) -> str: return self.__class__.__name__ + @final + @contextmanager + def locating_model_choice(self, *, field_name: str) -> Generator[None, None, None]: + """Run a model deck check, raising its refusal located on this pipe and on the field that names the model. + + Every operator that names a model checks it against the deck when it is built, through this one + wrapper, so an unknown model gives the same located ``PipeOperatorModelChoiceError`` whatever the + pipe type — which bundle validation turns into one ``unknown_model`` item. ``field_name`` is the + blueprint field the reference was written in (``model``, or ``model_to_structure`` on a + ``PipeLLM``), the key a fix rewrites. + """ + try: + yield + except ModelChoiceNotFoundError as model_choice_error: + raise PipeOperatorModelChoiceError.make_from_model_choice_not_found( + model_choice_error=model_choice_error, + pipe_type=self.class_name, + pipe_code=self.code, + domain_code=self.domain_code, + field_name=field_name, + ) from model_choice_error + @final @override async def _live_run_pipe( diff --git a/pipelex/pipe_operators/search/pipe_search.py b/pipelex/pipe_operators/search/pipe_search.py index 050f439f1b..219446bfa3 100644 --- a/pipelex/pipe_operators/search/pipe_search.py +++ b/pipelex/pipe_operators/search/pipe_search.py @@ -3,7 +3,6 @@ from typing_extensions import override from pipelex import log -from pipelex.cogt.exceptions import ModelChoiceNotFoundError from pipelex.cogt.models.model_deck_check import check_search_choice_with_deck from pipelex.cogt.search.search_setting import SearchModelChoice from pipelex.cogt.templating.template_blueprint import TemplateBlueprint @@ -51,11 +50,8 @@ def required_variables(self) -> set[str]: @override def validate_inputs_static(self): if self.search_choice: - try: + with self.locating_model_choice(field_name="model"): check_search_choice_with_deck(search_choice=self.search_choice) - except ModelChoiceNotFoundError as exc: - msg = f"Search choice '{self.search_choice}' was not found in the model deck" - raise ValueError(msg) from exc # Guard-lint (D7): every reference to a declared-optional input must be guarded. lint_optional_input_guards( diff --git a/pipelex/pipe_operators/structure/pipe_structure.py b/pipelex/pipe_operators/structure/pipe_structure.py index 599b60fef7..322d34e561 100644 --- a/pipelex/pipe_operators/structure/pipe_structure.py +++ b/pipelex/pipe_operators/structure/pipe_structure.py @@ -55,7 +55,8 @@ def required_variables(self) -> set[str]: @override def validate_inputs_static(self): if self.llm_choice is not None and not isinstance(self.llm_choice, LLMSetting): - check_llm_choice_with_deck(llm_choice=self.llm_choice) + with self.locating_model_choice(field_name="model"): + check_llm_choice_with_deck(llm_choice=self.llm_choice) @override def validate_inputs_with_library(self): diff --git a/pipelex/pipeline/exceptions.py b/pipelex/pipeline/exceptions.py index f0bf271f79..a8e75c66fe 100644 --- a/pipelex/pipeline/exceptions.py +++ b/pipelex/pipeline/exceptions.py @@ -112,7 +112,11 @@ def _summarize_bundle_validation_message( class ValidateBundleError(PipelexError): - """Raised when bundle validation fails. + """Raised when a bundle is refused while it is loaded or validated: the invalid verdict, carrying one + structured item per refusal in ``validation_errors``. Every refusal of the bundle itself becomes one — + the parser, factory and pipe-validation errors, a failing dry run, an unknown model (``unknown_model``) + and any other refusal of the caller's input — while a failure of the tool or its environment propagates + as a no-verdict fault instead. This error aggregates validation errors from different stages: - Blueprint validation errors (from interpreter) diff --git a/pipelex/pipeline/fixes/planner.py b/pipelex/pipeline/fixes/planner.py index a2dafb9810..adc9e88454 100644 --- a/pipelex/pipeline/fixes/planner.py +++ b/pipelex/pipeline/fixes/planner.py @@ -6,12 +6,13 @@ """ from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipesAndConceptValidationErrorData -from pipelex.suggested_fix import DeleteKeyOp, EnsureTableOp, FixOp, FixSafety, RenameTableKeyOp, SetKeyOp, SuggestedFix +from pipelex.suggested_fix import DeleteKeyOp, EnsureTableOp, FixOp, FixSafety, RemapValueOp, RenameTableKeyOp, SetKeyOp, SuggestedFix MATCH_SEQUENCE_OUTPUT_FIX_CODE = "match-sequence-output" SYNC_CONTROLLER_INPUTS_FIX_CODE = "sync-controller-inputs" STRIP_NATIVE_CONCEPT_REDECL_FIX_CODE = "strip-native-concept-redecl" STRIP_NAMESPACE_FIX_CODE = "strip-namespace" +RENAME_MODEL_FIX_CODE = "rename-model" # Every fix-rule code the planner can emit — the validation set for user-facing rule filters # (``--select`` / ``--ignore``). A new rule constant above must be added here; the CLI rejects @@ -22,6 +23,7 @@ SYNC_CONTROLLER_INPUTS_FIX_CODE, STRIP_NATIVE_CONCEPT_REDECL_FIX_CODE, STRIP_NAMESPACE_FIX_CODE, + RENAME_MODEL_FIX_CODE, } ) @@ -33,13 +35,47 @@ def plan_fix_for_pipe_validation_error(error_data: PipesAndConceptValidationErro the correct value — so the same error types raised elsewhere (PipeParallel / PipeCondition / operator pipes) carry no enrichment and are suppressed here structurally. """ + if error_data.error_type is None: + return None if error_data.error_type.is_inadequate_output: return _plan_match_sequence_output(error_data) if error_data.error_type.is_controller_input_drift: return _plan_sync_controller_inputs(error_data) + if error_data.error_type.is_unknown_model: + return _plan_rename_model(error_data=error_data) return None +def _plan_rename_model(*, error_data: PipesAndConceptValidationErrorData) -> SuggestedFix | None: + """``rename-model``: an ``unknown_model`` whose deck offers exactly one close match of the same kind + becomes a ``remap_value`` of the pipe's model field, from the reference as written to that match. + + A single match is the only case with one obvious correction; with several, choosing among them is + the author's call, and with none there is nothing to write. The remap names the reference as written, + so the op rewrites the field only while it still holds that exact value and leaves a field the + author has since edited alone. + """ + if error_data.pipe_code is None or error_data.field_name is None or error_data.model_reference is None: + return None + if error_data.suggestions is None or len(error_data.suggestions) != 1: + return None + suggestion = error_data.suggestions[0] + description = f"Replace model '{error_data.model_reference}' of pipe '{error_data.pipe_code}' with '{suggestion}'" + return SuggestedFix( + fix_code=RENAME_MODEL_FIX_CODE, + description=f"{description}, its one close match in the model deck", + safety=FixSafety.SAFE, + source=error_data.source, + ops=[ + RemapValueOp( + table_path=["pipe", error_data.pipe_code], + key=error_data.field_name, + mapping={error_data.model_reference: suggestion}, + ), + ], + ) + + def _plan_match_sequence_output(error_data: PipesAndConceptValidationErrorData) -> SuggestedFix | None: """``match-sequence-output``: an ``INADEQUATE_OUTPUT_*`` carrying the enriched ``expected_output_ref`` becomes a ``set_key`` of the pipe's ``output``. diff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.py index ef99249ad8..61287c0f8c 100644 --- a/pipelex/pipeline/validate_bundle.py +++ b/pipelex/pipeline/validate_bundle.py @@ -11,8 +11,16 @@ from typing_extensions import TypedDict from pipelex import log -from pipelex.base_exceptions import PipelexUnexpectedError -from pipelex.core.pipes.exceptions import PipeFactoryError, PipeRunError, PipeValidationError +from pipelex.base_exceptions import PipelexError, PipelexUnexpectedError, SecurityError, error_domain_is_input +from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipesAndConceptValidationErrorData +from pipelex.core.pipes.exceptions import ( + PipeFactoryError, + PipeLoadRefusalError, + PipeOperatorModelChoiceError, + PipeRunError, + PipeValidationError, + caller_facing_refusal_text, +) from pipelex.core.qualified_ref import QualifiedRef from pipelex.core.validation import report_validation_error from pipelex.interpreter_hub import ( @@ -28,6 +36,7 @@ from pipelex.mthds_parsing.exceptions import MthdsParserError from pipelex.mthds_parsing.handle_pipe_errors import ( categorize_pipe_factory_error, + categorize_pipe_operator_model_choice_error, categorize_pipe_validation_error, categorize_pipe_validation_with_libraries_error, ) @@ -122,11 +131,21 @@ def translate_to_validate_bundle_error() -> Generator[None, None, None]: Single source of truth for the bundle-loading error cascade, shared by the bundle-loading entry points: ``validate_bundle``, ``validate_bundles_from_directory``, - and ``pipelex.pipeline.resolve_bundle.resolve_crate_from_contents``. + ``pipelex.pipeline.resolve_bundle.resolve_crate_from_contents``, and the library loads and + dry-run sweeps of the ``validate pipe`` / ``validate --all`` commands. A ``MthdsParserError`` becomes a ``ValidateBundleError`` carrying the blueprint validation errors, a ``PipeFactoryError`` carries the categorized factory error, etc. Sharing one source of truth means a new handler only needs to be added once. + + **Every refusal of the bundle is a verdict.** After the class-specific arms, a final arm turns any + other ``PipelexError`` whose report is ``input``-domained (the caller's fault) into a verdict with one + item, so a refusal nobody wrote an arm for still reaches the author as an invalid bundle rather than + as a crash or a no-verdict fault. Only a failure of the tool or its environment — a ``config`` or + ``runtime`` fault, an unclassified one, anything that is not a ``PipelexError`` — propagates as + no verdict, along with the refusals that must keep their own class: ``PipeNotFoundError`` (its + dedicated not-found handler), a ``SecurityError`` (never absorbed by a domain handler), and a + ``ValidateBundleError`` already produced (never re-wrapped). """ try: yield @@ -206,6 +225,60 @@ def translate_to_validate_bundle_error() -> Generator[None, None, None]: message=dry_run_error.message, dry_run_error_message=dry_run_error.message, ) from dry_run_error + except PipeOperatorModelChoiceError as model_choice_error: + # A pipe names a model its deck does not define: the operator located it on the pipe and the + # field, and the library load added the file, so it becomes one `unknown_model` item carrying + # the reference as written, the model type and the deck's suggestions. + raise ValidateBundleError( + message=model_choice_error.message, + pipe_validation_errors=[categorize_pipe_operator_model_choice_error(model_choice_error=model_choice_error)], + ) from model_choice_error + except (ValidateBundleError, SecurityError): + # Both are PipelexErrors the general arm below would otherwise catch. A verdict already produced + # (e.g. by a nested load) passes through as it is: re-wrapping it would flatten its items into one. + # A security refusal is never absorbed into a domain answer, verdict included. + raise + except PipelexError as refusal: + verdict = _make_refusal_verdict(refusal=refusal) + if verdict is None: + raise + raise verdict from refusal + + +def _make_refusal_verdict(*, refusal: PipelexError) -> ValidateBundleError | None: + """The one-item verdict for a refusal of the caller's input that has no arm of its own, else ``None``. + + ``None`` — no verdict — unless the refusal's report is ``input``-domained: a ``config`` or ``runtime`` + fault, or an unclassified one, is a failure of the tool or its environment, which the validator must + not report as the bundle's fault. The item is ``pipe_validation`` when the library load located the + refusal on the pipe it was building (``PipeLoadRefusalError``, with the pipe's code, domain and file), + and ``blueprint_validation`` otherwise. It carries no ``error_type``: a refusal with a closed code + has its own arm above, and naming one here would claim a diagnosis the refusal does not make. + + The item's text is the refusal's message only when that message was authored as caller-facing copy, + and otherwise its title: the verdict is caller-facing as a whole and is kept verbatim under STRICT + disclosure, so internal text must not ride it past the redaction it would otherwise get. + """ + if not error_domain_is_input(refusal.to_error_report().error_domain): + return None + message = caller_facing_refusal_text(refusal=refusal) + if isinstance(refusal, PipeLoadRefusalError): + return ValidateBundleError( + message=message, + pipe_validation_errors=[ + PipesAndConceptValidationErrorData( + pipe_code=refusal.pipe_code, + domain_code=refusal.domain_code, + source=refusal.source, + message=message, + field_path="", + ) + ], + ) + return ValidateBundleError( + message=message, + pipelex_bundle_blueprint_validation_errors=[PipelexBundleBlueprintValidationErrorData(message=message)], + ) def _pipes_to_dry_run(loaded_pipes: list[PipeAbstract], *, dry_run_pipe_codes: list[str] | None) -> list[PipeAbstract]: diff --git a/pipelex/pipeline/validation_errors.py b/pipelex/pipeline/validation_errors.py index c5939e1d11..7dec304866 100644 --- a/pipelex/pipeline/validation_errors.py +++ b/pipelex/pipeline/validation_errors.py @@ -120,6 +120,9 @@ def build_validation_error_items( field_path=pipe_error.field_path or None, field_name=pipe_error.field_name, variable_names=pipe_error.variable_names or None, + model_reference=pipe_error.model_reference, + model_type=pipe_error.model_type, + suggestions=pipe_error.suggestions or None, message=pipe_error.message, suggested_fix=plan_fix_for_pipe_validation_error(pipe_error), ) diff --git a/pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/bundle.mthds b/pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/bundle.mthds new file mode 100644 index 0000000000..9431b0e515 --- /dev/null +++ b/pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/bundle.mthds @@ -0,0 +1,12 @@ +domain = "tide_tables" +description = "Write the note telling harbour users when the tide turns" +main_pipe = "write_tide_note" + +# The model named here is one no model deck defines, so the pipe is refused when it is built. +[pipe.write_tide_note] +type = "PipeLLM" +description = "Write the tide note for the harbour board" +inputs = { tide_times = "Text" } +output = "Text" +model = "no-such-model-handle" +prompt = "Write a short note for the harbour board from these tide times: $tide_times" diff --git a/pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/entry.toml b/pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/entry.toml new file mode 100644 index 0000000000..51f0889acf --- /dev/null +++ b/pipelex/test_extras/mthds_corpus/entries/invalid_unknown_model/entry.toml @@ -0,0 +1,12 @@ +name = "invalid_unknown_model" +description = "A harbour tide note whose writing pipe names a model that no model deck defines" + +validity = "invalid" +tier = "static" +granularity = "focused" + +covers = [ + "error.unknown_model", +] + +expected_error = "unknown_model" diff --git a/pipelex/test_extras/mthds_corpus/vocabulary.toml b/pipelex/test_extras/mthds_corpus/vocabulary.toml index 441b1b6377..06d179290d 100644 --- a/pipelex/test_extras/mthds_corpus/vocabulary.toml +++ b/pipelex/test_extras/mthds_corpus/vocabulary.toml @@ -177,6 +177,10 @@ fails_at = "runtime" code = "unresolved_pipe_dependency" fails_at = "runtime" +[error.unknown_model] +code = "unknown_model" +fails_at = "runtime" + [error.unknown_validation_error] code = "unknown_validation_error" excluded = "The generic fallback for a validation failure that matched no specific code. Reaching it on purpose would mean authoring a bundle that breaks in a way the categorizer does not recognize — which is a state to fix in the runtime, not to pin in a corpus entry that would then have to be rewritten the moment the fault gets a code of its own." diff --git a/pipelex/validation_error_types.py b/pipelex/validation_error_types.py index 69a4b64a86..28ef3d4ba7 100644 --- a/pipelex/validation_error_types.py +++ b/pipelex/validation_error_types.py @@ -91,6 +91,11 @@ class PipeValidationErrorType(StrEnum): UNRESOLVED_CONCEPT = "unresolved_concept" UNRESOLVED_PIPE_DEPENDENCY = "unresolved_pipe_dependency" + # A pipe's model field names a handle, alias, preset or waterfall its model deck does not define, + # refused when the pipe is built. The item carries the field's path, the reference as written, the + # model type and the deck's close matches, so the author can pick one. + UNKNOWN_MODEL = "unknown_model" + # Generic fallback for unexpected validation errors UNKNOWN_VALIDATION_ERROR = "unknown_validation_error" @@ -123,6 +128,7 @@ def is_controller_input_drift(self) -> bool: | PipeValidationErrorType.NATIVE_CONCEPT_REDECLARATION | PipeValidationErrorType.UNRESOLVED_CONCEPT | PipeValidationErrorType.UNRESOLVED_PIPE_DEPENDENCY + | PipeValidationErrorType.UNKNOWN_MODEL | PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR ): return False @@ -153,6 +159,7 @@ def is_inadequate_output(self) -> bool: | PipeValidationErrorType.NATIVE_CONCEPT_REDECLARATION | PipeValidationErrorType.UNRESOLVED_CONCEPT | PipeValidationErrorType.UNRESOLVED_PIPE_DEPENDENCY + | PipeValidationErrorType.UNKNOWN_MODEL | PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR ): return False @@ -190,6 +197,7 @@ def is_inadequate_output_multiplicity(self) -> bool: | PipeValidationErrorType.NATIVE_CONCEPT_REDECLARATION | PipeValidationErrorType.UNRESOLVED_CONCEPT | PipeValidationErrorType.UNRESOLVED_PIPE_DEPENDENCY + | PipeValidationErrorType.UNKNOWN_MODEL | PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR ): return False @@ -221,6 +229,7 @@ def is_native_concept_redeclaration(self) -> bool: | PipeValidationErrorType.INPUT_PRESENCE_VACUOUS | PipeValidationErrorType.UNRESOLVED_CONCEPT | PipeValidationErrorType.UNRESOLVED_PIPE_DEPENDENCY + | PipeValidationErrorType.UNKNOWN_MODEL | PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR ): return False @@ -257,6 +266,39 @@ def is_invalid_pipe_code_syntax(self) -> bool: | PipeValidationErrorType.NATIVE_CONCEPT_REDECLARATION | PipeValidationErrorType.UNRESOLVED_CONCEPT | PipeValidationErrorType.UNRESOLVED_PIPE_DEPENDENCY + | PipeValidationErrorType.UNKNOWN_MODEL + | PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR + ): + return False + + @property + def is_unknown_model(self) -> bool: + """True for the unknown-model refusal, which the fix planner renames when the deck offers one close match.""" + match self: + case PipeValidationErrorType.UNKNOWN_MODEL: + return True + case ( + PipeValidationErrorType.MISSING_INPUT_VARIABLE + | PipeValidationErrorType.EXTRANEOUS_INPUT_VARIABLE + | PipeValidationErrorType.INPUT_STUFF_SPEC_MISMATCH + | PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT + | PipeValidationErrorType.INADEQUATE_OUTPUT_MULTIPLICITY + | PipeValidationErrorType.CIRCULAR_DEPENDENCY_ERROR + | PipeValidationErrorType.LLM_OUTPUT_CANNOT_BE_IMAGE + | PipeValidationErrorType.INVALID_PIPE_CODE_SYNTAX + | PipeValidationErrorType.UNKNOWN_PIPE_TYPE + | PipeValidationErrorType.MISSING_PIPE_TYPE + | PipeValidationErrorType.BATCH_ITEM_NAME_COLLISION + | PipeValidationErrorType.OPTIONAL_MARKER_INVALID + | PipeValidationErrorType.OPTIONAL_NOT_HANDLED + | PipeValidationErrorType.OPTIONAL_OUTPUT_REQUIRED + | PipeValidationErrorType.OPTIONAL_INPUT_UNGUARDED + | PipeValidationErrorType.OPTIONAL_BRANCH_REQUIRED_FIELD + | PipeValidationErrorType.OPTIONAL_FORCE_REDUNDANT + | PipeValidationErrorType.INPUT_PRESENCE_VACUOUS + | PipeValidationErrorType.NATIVE_CONCEPT_REDECLARATION + | PipeValidationErrorType.UNRESOLVED_CONCEPT + | PipeValidationErrorType.UNRESOLVED_PIPE_DEPENDENCY | PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR ): return False diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txt index 34d8579917..87aafe4d13 100644 --- a/tests/data/errors/error_identity.txt +++ b/tests/data/errors/error_identity.txt @@ -227,6 +227,7 @@ PipeInputsFactoryError | Pipe inputs factory | https://docs.pipelex.com/latest/e PipeJobError | Pipe job | https://docs.pipelex.com/latest/errors/pipe-job-error/ PipeLLMFactoryError | Pipe LLM factory | https://docs.pipelex.com/latest/errors/pipe-llm-factory-error/ PipeLibraryError | Pipe library | https://docs.pipelex.com/latest/errors/pipe-library-error/ +PipeLoadRefusalError | Pipe load refusal | https://docs.pipelex.com/latest/errors/pipe-load-refusal-error/ PipeLoadingError | Pipe loading | https://docs.pipelex.com/latest/errors/pipe-loading-error/ PipeNotFoundError | Pipe not found | https://docs.pipelex.com/latest/errors/pipe-not-found-error/ PipeOperatorModelAvailabilityError | Pipe operator model availability | https://docs.pipelex.com/latest/errors/pipe-operator-model-availability-error/ diff --git a/tests/integration/pipelex/cli/test_validate_load_refusals_cli.py b/tests/integration/pipelex/cli/test_validate_load_refusals_cli.py new file mode 100644 index 0000000000..9fc51c3c83 --- /dev/null +++ b/tests/integration/pipelex/cli/test_validate_load_refusals_cli.py @@ -0,0 +1,174 @@ +"""Pin: the local CLIs answer a load-time refusal and a failing dry run as a negative verdict (exit 1). + +On the validate surface exit 1 is a produced negative verdict and exit 2 is "no verdict could be +produced". These tests drive the real validation engine through the command cores: + +- ``pipelex validate bundle`` on a bundle naming an unknown model prints the grouped invalid-bundle + panel with the ``unknown_model`` item and exits 1, with no traceback; +- ``pipelex-agent validate bundle`` on the same bundle emits its invalid-verdict envelope + (``is_valid: false`` and the item) and exits 1, where it used to answer the no-verdict envelope with + exit 2; +- ``pipelex validate pipe `` on a pipe whose dry run fails, and ``pipelex validate --all`` on the + same library, print the invalid-bundle panel and exit 1, where they used to print a traceback; +- ``pipelex validate pipe `` on a library naming an unknown model exits 1 with the item. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import TYPE_CHECKING + +import pytest +import typer +from rich.console import Console + +from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, set_agent_cli_error_format +from pipelex.cli.agent_cli.commands.validate.bundle_cmd import validate_bundle_cmd as agent_validate_bundle_cmd +from pipelex.cli.commands.validate._validate_core import ( + _validate_pipe_or_bundle, # pyright: ignore[reportPrivateUsage] + do_validate_all_libraries_and_dry_run, +) +from pipelex.interpreter_hub import clear_current_library, get_current_library_id_or_none, get_library_manager, set_current_library +from pipelex.test_extras.mthds_corpus.loader import get_entry + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from pytest_mock import MockerFixture + +_UNKNOWN_MODEL_BUNDLE = """ +domain = "tide_tables" +description = "Write the note telling harbour users when the tide turns" +main_pipe = "write_tide_note" + +[pipe.write_tide_note] +type = "PipeLLM" +description = "Write the tide note for the harbour board" +inputs = { tide_times = "Text" } +output = "Text" +model = "gpt-5.1" +prompt = "Write a short note for the harbour board from these tide times: $tide_times" +""" + +# The corpus's dry-run residual: a PipeCondition whose rule renders to nothing, so its dry run fails. +_DRY_RUN_FAILURE_ENTRY = "invalid_dry_run_residual" +_DRY_RUN_FAILURE_PIPE = "route_parcel" + + +@pytest.fixture +def console(mocker: MockerFixture) -> Console: + """A recording console patched into the error handlers, so the panel can be read back.""" + recorded_console = Console(width=200, record=True, color_system=None) + mocker.patch("pipelex.cli.error_handlers.get_console", return_value=recorded_console) + return recorded_console + + +@pytest.fixture +def restore_current_library() -> Iterator[None]: + """The pipe and --all paths leave their library open and current (the command's teardown owns it).""" + outer_library_id = get_current_library_id_or_none() + yield + current_library_id = get_current_library_id_or_none() + if current_library_id is not None and current_library_id != outer_library_id: + get_library_manager().teardown(library_id=current_library_id) + if outer_library_id is not None: + set_current_library(library_id=outer_library_id) + else: + clear_current_library() + + +@pytest.fixture +def unknown_model_bundle(tmp_path: Path) -> Path: + bundle_path = tmp_path / "bundle.mthds" + bundle_path.write_text(_UNKNOWN_MODEL_BUNDLE, encoding="utf-8") + return bundle_path + + +class TestValidateLoadRefusalsCli: + def test_bare_validate_bundle_renders_the_unknown_model_as_an_invalid_bundle(self, console: Console, unknown_model_bundle: Path) -> None: + with pytest.raises(typer.Exit) as exc_info: + asyncio.run(_validate_pipe_or_bundle(bundle_path=unknown_model_bundle, library_dirs=[unknown_model_bundle.parent])) + + assert exc_info.value.exit_code == 1 + output = console.export_text() + assert "Bundle validation failed" in output + assert "Pipe Validation Errors:" in output + assert "Unknown Model" in output + assert "Model handle 'gpt-5.1' was not found in the model deck" in output + assert "Path: pipe.write_tide_note.model" in output + assert "Traceback" not in output + + def test_agent_validate_bundle_emits_the_invalid_verdict_envelope( + self, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + unknown_model_bundle: Path, + ) -> None: + mocker.patch("pipelex.cli.agent_cli.commands.validate.bundle_cmd.make_pipelex_for_agent_cli") + mocker.patch("pipelex.cli.agent_cli.commands.validate.bundle_cmd.Pipelex.teardown_if_needed") + try: + with pytest.raises(typer.Exit) as exc_info: + agent_validate_bundle_cmd( + path=str(unknown_model_bundle), + library_dir=[str(unknown_model_bundle.parent)], + output_format=CliOutputFormat.JSON, + ) + finally: + set_agent_cli_error_format(CliOutputFormat.JSON) + + assert exc_info.value.exit_code == 1 + envelope = json.loads(capsys.readouterr().err) + assert envelope["is_valid"] is False + assert envelope["error_type"] == "ValidateBundleError" + assert envelope["error_domain"] == "input" + (item,) = envelope["validation_errors"] + assert item["category"] == "pipe_validation" + assert item["error_type"] == "unknown_model" + assert item["pipe_code"] == "write_tide_note" + assert item["domain_code"] == "tide_tables" + assert item["source"] == str(unknown_model_bundle) + assert item["field_path"] == "pipe.write_tide_note.model" + assert item["model_reference"] == "gpt-5.1" + assert item["model_type"] == "llm" + assert item["suggestions"] + + @pytest.mark.usefixtures("restore_current_library") + def test_bare_validate_pipe_renders_a_failing_dry_run_as_an_invalid_bundle(self, console: Console) -> None: + entry = get_entry(name=_DRY_RUN_FAILURE_ENTRY) + + with pytest.raises(typer.Exit) as exc_info: + asyncio.run(_validate_pipe_or_bundle(pipe_code=_DRY_RUN_FAILURE_PIPE, library_dirs=[entry.directory])) + + assert exc_info.value.exit_code == 1 + output = console.export_text() + assert "Bundle validation failed" in output + assert "Dry Run Error:" in output + assert "Traceback" not in output + + @pytest.mark.usefixtures("restore_current_library") + def test_bare_validate_all_renders_a_failing_dry_run_as_an_invalid_bundle(self, console: Console) -> None: + entry = get_entry(name=_DRY_RUN_FAILURE_ENTRY) + + with pytest.raises(typer.Exit) as exc_info: + do_validate_all_libraries_and_dry_run(library_dirs=[entry.directory]) + + assert exc_info.value.exit_code == 1 + output = console.export_text() + assert "Bundle validation failed" in output + assert "Dry Run Error:" in output + assert "Traceback" not in output + + @pytest.mark.usefixtures("restore_current_library") + def test_bare_validate_pipe_renders_an_unknown_model_in_the_library_as_an_invalid_bundle( + self, console: Console, unknown_model_bundle: Path + ) -> None: + with pytest.raises(typer.Exit) as exc_info: + asyncio.run(_validate_pipe_or_bundle(pipe_code="write_tide_note", library_dirs=[unknown_model_bundle.parent])) + + assert exc_info.value.exit_code == 1 + output = console.export_text() + assert "Unknown Model" in output + assert "Path: pipe.write_tide_note.model" in output + assert "Traceback" not in output diff --git a/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py new file mode 100644 index 0000000000..a257c18f3e --- /dev/null +++ b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py @@ -0,0 +1,320 @@ +"""Pin: every refusal raised while loading a bundle is a verdict item, never a no-verdict fault. + +A bundle validator answers either a verdict (valid, or invalid with located items) or "no verdict could +be produced", which is reserved for a failure of the tool or its environment. These tests load real +bundles through ``validate_bundle`` and pin the verdict each refusal produces: + +- **The unknown model, per pipe type and per reference kind.** A pipe whose model field names a handle, + an alias or a preset the deck does not define validates to one ``pipe_validation`` item with the + closed ``error_type`` ``unknown_model``, carrying the pipe, the domain, the source file, the field + path, the reference as written, the model type and the deck's suggestions, and a ``rename-model`` fix + when the deck offers exactly one suggestion. Every pipe type that names a model gives the same item + (``PipeExtract`` and ``PipeSearch`` used to turn it into an ``unknown_validation_error`` whose message + was a Python repr). +- **The general arm.** Any other ``input``-domained refusal raised while building a pipe validates to one + item located on the pipe and its file, keeping its message only when it is caller-facing; a + ``config``-domained fault raised at the same place still propagates as no verdict. +- **The run path.** Setting up a run on the unknown-model bundle refuses it with an error naming the + pipe, whose STRICT projection is still an HTTP 422 carrying the deck check's sentence. + +The deck is the test session's own, so the suggestion lists are read off it rather than pinned in +full, except for the one-suggestion alias whose rename fix is the point of its case. +""" + +from pathlib import Path +from typing import ClassVar, NamedTuple + +import pytest +from pytest_mock import MockerFixture + +from pipelex.base_exceptions import DisclosureMode, ErrorDomain, PipelexError, ValidationErrorCategory, ValidationErrorItem +from pipelex.config import get_config +from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError +from pipelex.pipeline.exceptions import ValidateBundleError +from pipelex.pipeline.fixes.planner import RENAME_MODEL_FIX_CODE +from pipelex.pipeline.pipeline_run_setup import pipeline_run_setup +from pipelex.pipeline.validate_bundle import validate_bundle +from pipelex.suggested_fix import RemapValueOp +from pipelex.system.pipe_run_mode import PipeRunMode +from pipelex.validation_error_types import PipeValidationErrorType + +_DOMAIN = "tide_tables" + + +def _llm_bundle(*, model_line: str) -> str: + return f""" +domain = "{_DOMAIN}" +description = "Write the note telling harbour users when the tide turns" +main_pipe = "write_tide_note" + +[pipe.write_tide_note] +type = "PipeLLM" +description = "Write the tide note for the harbour board" +inputs = {{ tide_times = "Text" }} +output = "Text" +{model_line} +prompt = "Write a short note for the harbour board from these tide times: $tide_times" +""" + + +_IMG_GEN_BUNDLE = f""" +domain = "{_DOMAIN}" +description = "Draw the board announcing the tide times" +main_pipe = "draw_tide_board" + +[pipe.draw_tide_board] +type = "PipeImgGen" +description = "Draw the painted board announcing the tide times" +output = "Image" +model = "nano-banana-9" +prompt = "A painted wooden harbour board announcing the tide times, morning light" +""" + +_EXTRACT_BUNDLE = f""" +domain = "{_DOMAIN}" +description = "Read the pages of a printed tide almanac" +main_pipe = "read_almanac_pages" + +[pipe.read_almanac_pages] +type = "PipeExtract" +description = "Read each page of a scanned tide almanac" +inputs = {{ almanac_scan = "Document" }} +output = "Page[]" +model = "@default-extrct" +""" + +_SEARCH_BUNDLE = f""" +domain = "{_DOMAIN}" +description = "Look up the tide times of a harbour" +main_pipe = "look_up_tides" + +[pipe.look_up_tides] +type = "PipeSearch" +description = "Look up today's tide times for the harbour" +inputs = {{ harbour_name = "Text" }} +output = "SearchResult" +model = "@default-serch" +prompt = "What are today's tide times at $harbour_name?" +""" + + +class _UnknownModelCase(NamedTuple): + case_id: str + bundle: str + pipe_code: str + field_name: str + model_reference: str + model_type: str + model_sentence: str + + +_UNKNOWN_MODEL_CASES: list[_UnknownModelCase] = [ + _UnknownModelCase( + case_id="llm_handle", + bundle=_llm_bundle(model_line='model = "gpt-5.1"'), + pipe_code="write_tide_note", + field_name="model", + model_reference="gpt-5.1", + model_type="llm", + model_sentence="Model handle 'gpt-5.1' was not found in the model deck", + ), + _UnknownModelCase( + case_id="llm_alias", + bundle=_llm_bundle(model_line='model = "@best-sonet"'), + pipe_code="write_tide_note", + field_name="model", + model_reference="@best-sonet", + model_type="llm", + model_sentence="Alias 'best-sonet' was not found in the model deck", + ), + _UnknownModelCase( + case_id="llm_preset", + bundle=_llm_bundle(model_line='model = "$writting-factual"'), + pipe_code="write_tide_note", + field_name="model", + model_reference="$writting-factual", + model_type="llm", + model_sentence="LLM preset 'writting-factual' was not found in the model deck", + ), + _UnknownModelCase( + case_id="llm_model_to_structure", + bundle=_llm_bundle(model_line='model_to_structure = "@best-sonet"'), + pipe_code="write_tide_note", + field_name="model_to_structure", + model_reference="@best-sonet", + model_type="llm", + model_sentence="Alias 'best-sonet' was not found in the model deck", + ), + _UnknownModelCase( + case_id="img_gen_handle", + bundle=_IMG_GEN_BUNDLE, + pipe_code="draw_tide_board", + field_name="model", + model_reference="nano-banana-9", + model_type="img_gen", + model_sentence="Model handle 'nano-banana-9' was not found in the model deck", + ), + _UnknownModelCase( + case_id="extract_alias", + bundle=_EXTRACT_BUNDLE, + pipe_code="read_almanac_pages", + field_name="model", + model_reference="@default-extrct", + model_type="text_extractor", + model_sentence="Alias 'default-extrct' was not found in the model deck", + ), + _UnknownModelCase( + case_id="search_alias", + bundle=_SEARCH_BUNDLE, + pipe_code="look_up_tides", + field_name="model", + model_reference="@default-serch", + model_type="search", + model_sentence="Alias 'default-serch' was not found in the model deck", + ), +] + + +def _write_bundle(*, directory: Path, content: str) -> Path: + bundle_path = directory / "bundle.mthds" + bundle_path.write_text(content, encoding="utf-8") + return bundle_path + + +async def _single_item(*, bundle_path: Path) -> ValidationErrorItem: + with pytest.raises(ValidateBundleError) as raised: + await validate_bundle(mthds_file_path=bundle_path, library_dirs=[bundle_path.parent]) + items = raised.value.to_error_report().validation_errors or [] + assert len(items) == 1, f"expected exactly one item, got {items!r}" + return items[0] + + +class _CallerFacingRefusalError(PipelexError): + """An input refusal whose message was authored for the caller — stands in for any future one.""" + + error_domain = ErrorDomain.INPUT + _authors_caller_facing_message = True + + +class _InternalInputRefusalError(PipelexError): + """An input refusal whose message is internal text the verdict must not carry.""" + + error_domain = ErrorDomain.INPUT + _declared_title: ClassVar[str | None] = "Harbour board refusal" + + +class _ConfigFaultError(PipelexError): + """A fault of the environment, which is no verdict about the bundle.""" + + error_domain = ErrorDomain.CONFIG + + +@pytest.mark.asyncio(loop_scope="class") +class TestValidateBundleLoadRefusals: + @pytest.mark.parametrize("case", _UNKNOWN_MODEL_CASES, ids=[case.case_id for case in _UNKNOWN_MODEL_CASES]) + async def test_unknown_model_is_one_located_unknown_model_item(self, case: _UnknownModelCase, tmp_path: Path) -> None: + bundle_path = _write_bundle(directory=tmp_path, content=case.bundle) + + item = await _single_item(bundle_path=bundle_path) + + assert item.category == ValidationErrorCategory.PIPE_VALIDATION + assert item.error_type == PipeValidationErrorType.UNKNOWN_MODEL + assert item.pipe_code == case.pipe_code + assert item.domain_code == _DOMAIN + assert item.source == str(bundle_path) + assert item.field_name == case.field_name + assert item.field_path == f"pipe.{case.pipe_code}.{case.field_name}" + assert item.model_reference == case.model_reference + assert item.model_type == case.model_type + assert case.model_sentence in item.message + assert f"Pipe '{case.pipe_code}'" in item.message + # The suggestions ride as a list and stay in the message, for the consumers that keep only it. + assert item.suggestions, f"the deck offers no close match for {case.model_reference!r}" + for suggestion in item.suggestions: + assert suggestion in item.message + + async def test_one_suggestion_carries_a_rename_fix(self, tmp_path: Path) -> None: + bundle_path = _write_bundle(directory=tmp_path, content=_llm_bundle(model_line='model = "@best-sonet"')) + + item = await _single_item(bundle_path=bundle_path) + + assert item.suggestions == ["@best-gpt"] + fix = item.suggested_fix + assert fix is not None + assert fix.fix_code == RENAME_MODEL_FIX_CODE + assert fix.safety.is_safe + assert fix.source == str(bundle_path) + assert fix.ops == [RemapValueOp(table_path=["pipe", "write_tide_note"], key="model", mapping={"@best-sonet": "@best-gpt"})] + + async def test_several_suggestions_carry_no_fix(self, tmp_path: Path) -> None: + bundle_path = _write_bundle(directory=tmp_path, content=_llm_bundle(model_line='model = "$writting-factual"')) + + item = await _single_item(bundle_path=bundle_path) + + assert item.suggestions is not None + assert len(item.suggestions) > 1 + assert item.suggested_fix is None + + async def test_caller_facing_input_refusal_at_load_is_a_located_item(self, tmp_path: Path, mocker: MockerFixture) -> None: + mocker.patch( + "pipelex.pipe_operators.llm.pipe_llm.check_llm_choice_with_deck", + side_effect=_CallerFacingRefusalError("The harbour board cannot show tides this far upriver."), + ) + bundle_path = _write_bundle(directory=tmp_path, content=_llm_bundle(model_line='model = "@best-gpt"')) + + item = await _single_item(bundle_path=bundle_path) + + assert item.category == ValidationErrorCategory.PIPE_VALIDATION + assert item.error_type is None + assert item.pipe_code == "write_tide_note" + assert item.domain_code == _DOMAIN + assert item.source == str(bundle_path) + assert "The harbour board cannot show tides this far upriver." in item.message + + async def test_internal_input_refusal_carries_its_title_not_its_message(self, tmp_path: Path, mocker: MockerFixture) -> None: + mocker.patch( + "pipelex.pipe_operators.llm.pipe_llm.check_llm_choice_with_deck", + side_effect=_InternalInputRefusalError("internal detail: registry slot 7 is stale"), + ) + bundle_path = _write_bundle(directory=tmp_path, content=_llm_bundle(model_line='model = "@best-gpt"')) + + item = await _single_item(bundle_path=bundle_path) + + assert item.pipe_code == "write_tide_note" + assert item.source == str(bundle_path) + assert "Harbour board refusal" in item.message + assert "registry slot 7" not in item.message + + async def test_config_fault_at_load_stays_no_verdict(self, tmp_path: Path, mocker: MockerFixture) -> None: + fault = _ConfigFaultError("the model deck could not be read") + mocker.patch("pipelex.pipe_operators.llm.pipe_llm.check_llm_choice_with_deck", side_effect=fault) + bundle_path = _write_bundle(directory=tmp_path, content=_llm_bundle(model_line='model = "@best-gpt"')) + + with pytest.raises(_ConfigFaultError) as raised: + await validate_bundle(mthds_file_path=bundle_path, library_dirs=[bundle_path.parent]) + + assert raised.value is fault + + async def test_run_setup_refuses_the_unknown_model_naming_the_pipe(self) -> None: + execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False, mock_inputs=True) + + with pytest.raises(PipeOperatorModelChoiceError) as raised: + await pipeline_run_setup( + storage_scope="test/scope", + user_id="test-user", + execution_config=execution_config, + mthds_contents=[_llm_bundle(model_line='model = "gpt-5.1"')], + pipe_code="write_tide_note", + pipe_run_mode=PipeRunMode.DRY, + ) + + error = raised.value + assert error.pipe_code == "write_tide_note" + assert error.field_name == "model" + report = error.to_error_report() + assert report.http_status == 422 + strict_payload = report.to_dict(disclosure_mode=DisclosureMode.STRICT) + assert strict_payload["error_type"] == "PipeOperatorModelChoiceError" + assert strict_payload["error_domain"] == "input" + assert "Pipe 'write_tide_note'" in strict_payload["message"] + assert "Model handle 'gpt-5.1' was not found in the model deck" in strict_payload["message"] diff --git a/tests/unit/pipelex/cli/test_agent_output.py b/tests/unit/pipelex/cli/test_agent_output.py index e7799afb1b..ff0a9c4899 100644 --- a/tests/unit/pipelex/cli/test_agent_output.py +++ b/tests/unit/pipelex/cli/test_agent_output.py @@ -369,7 +369,7 @@ def test_agent_error_error_domain_and_category_coexist(self, capsys: pytest.Capt """ # An error_type in AGENT_ERROR_DOMAINS that is NOT a CogtError subclass, so no derived # domain can pre-empt the lookup. - error_type = "PipeOperatorModelChoiceError" + error_type = "PipeOperatorModelAvailabilityError" assert error_type in AGENT_ERROR_DOMAINS, "precondition: error_type must be in AGENT_ERROR_DOMAINS" cause = CogtError("model not found", error_category=InferenceErrorCategory.UNKNOWN) diff --git a/tests/unit/pipelex/cli/test_error_handlers.py b/tests/unit/pipelex/cli/test_error_handlers.py index b70fd9bd2d..b054aeb457 100644 --- a/tests/unit/pipelex/cli/test_error_handlers.py +++ b/tests/unit/pipelex/cli/test_error_handlers.py @@ -37,6 +37,8 @@ def test_handle_model_choice_error_exits_and_calls_to_error_report(self, mocker: message="model 'gpt-5' not found", pipe_type="llm_text", pipe_code="my_pipe", + domain_code="my_domain", + field_name="model", model_type=ModelType.LLM, model_choice="gpt-5", ) diff --git a/tests/unit/pipelex/cli/test_error_handlers_snapshot.py b/tests/unit/pipelex/cli/test_error_handlers_snapshot.py index 2e01f46825..1cca0f86e6 100644 --- a/tests/unit/pipelex/cli/test_error_handlers_snapshot.py +++ b/tests/unit/pipelex/cli/test_error_handlers_snapshot.py @@ -78,6 +78,8 @@ def test_model_choice_error_panel_snapshot(self, mocker: MockerFixture) -> None: message="model 'gpt-5' not found", pipe_type="llm_text", pipe_code="my_pipe", + domain_code="my_domain", + field_name="model", model_type=ModelType.LLM, model_choice="gpt-5", ) diff --git a/tests/unit/pipelex/cli/test_run_core_wrapper.py b/tests/unit/pipelex/cli/test_run_core_wrapper.py index 4877b5856e..e659649ba7 100644 --- a/tests/unit/pipelex/cli/test_run_core_wrapper.py +++ b/tests/unit/pipelex/cli/test_run_core_wrapper.py @@ -95,6 +95,8 @@ def test_model_choice_error_dispatched_to_handler(self, wrapper_mocks: dict[str, message="model 'gpt-5' not found", pipe_type="llm_text", pipe_code="test_pipe", + domain_code="my_domain", + field_name="model", model_type=ModelType.LLM, model_choice="gpt-5", ) diff --git a/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py b/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py index 999946ed6b..273ad8e3c2 100644 --- a/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py +++ b/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py @@ -11,8 +11,8 @@ import pytest from pipelex.core.exceptions import PipesAndConceptValidationErrorData -from pipelex.pipeline.fixes.planner import plan_fix_for_pipe_validation_error -from pipelex.suggested_fix import DeleteKeyOp, EnsureTableOp, FixSafety, SetKeyOp +from pipelex.pipeline.fixes.planner import RENAME_MODEL_FIX_CODE, plan_fix_for_pipe_validation_error +from pipelex.suggested_fix import DeleteKeyOp, EnsureTableOp, FixSafety, RemapValueOp, SetKeyOp from pipelex.validation_error_types import PipeValidationErrorType # The inputs table of the pipe every input-drift case below is built around. @@ -57,6 +57,26 @@ def _input_drift_error_data( ) +def _unknown_model_error_data( + *, + suggestions: list[str] | None, + field_name: str | None = "model", + model_reference: str | None = "@best-sonet", +) -> PipesAndConceptValidationErrorData: + return PipesAndConceptValidationErrorData( + error_type=PipeValidationErrorType.UNKNOWN_MODEL, + domain_code="tide_tables", + source="main.mthds", + pipe_code="write_tide_note", + field_name=field_name, + message="unknown model", + field_path="pipe.write_tide_note.model", + model_reference=model_reference, + model_type="llm", + suggestions=suggestions, + ) + + class TestFixPlanner: def test_concept_mismatch_yields_match_sequence_output_fix(self) -> None: """An enriched INADEQUATE_OUTPUT_CONCEPT yields a SAFE set_key fix on the pipe's output.""" @@ -189,3 +209,33 @@ def test_input_drift_without_pipe_code_yields_none(self) -> None: """Without a pipe locator there is no TOML table to patch → no fix.""" fix = plan_fix_for_pipe_validation_error(_input_drift_error_data(pipe_code=None, expected_inputs={"text": "Text"}, declared_inputs={})) assert fix is None + + def test_unknown_model_with_one_suggestion_yields_rename_model_fix(self) -> None: + """One close match in the deck is the one obvious correction: remap the field from the reference as written.""" + fix = plan_fix_for_pipe_validation_error(_unknown_model_error_data(suggestions=["@best-gpt"])) + assert fix is not None + assert fix.fix_code == RENAME_MODEL_FIX_CODE + assert fix.safety == FixSafety.SAFE + assert fix.source == "main.mthds" + assert fix.ops == [RemapValueOp(table_path=["pipe", "write_tide_note"], key="model", mapping={"@best-sonet": "@best-gpt"})] + assert "'@best-sonet'" in fix.description + assert "'@best-gpt'" in fix.description + + def test_unknown_model_fix_rewrites_the_field_that_named_it(self) -> None: + fix = plan_fix_for_pipe_validation_error(_unknown_model_error_data(suggestions=["@best-gpt"], field_name="model_to_structure")) + assert fix is not None + assert fix.ops == [RemapValueOp(table_path=["pipe", "write_tide_note"], key="model_to_structure", mapping={"@best-sonet": "@best-gpt"})] + + @pytest.mark.parametrize("suggestions", [None, [], ["$writing-factual", "$writing-creative"]]) + def test_unknown_model_without_exactly_one_suggestion_yields_none(self, suggestions: list[str] | None) -> None: + """No match leaves nothing to write, and several leave the choice to the author.""" + assert plan_fix_for_pipe_validation_error(_unknown_model_error_data(suggestions=suggestions)) is None + + def test_unknown_model_without_its_locators_yields_none(self) -> None: + assert plan_fix_for_pipe_validation_error(_unknown_model_error_data(suggestions=["@best-gpt"], field_name=None)) is None + assert plan_fix_for_pipe_validation_error(_unknown_model_error_data(suggestions=["@best-gpt"], model_reference=None)) is None + + def test_uncoded_pipe_error_yields_none(self) -> None: + """A located refusal with no closed code (the cascade's general arm) has no fix to plan.""" + error_data = PipesAndConceptValidationErrorData(pipe_code="write_tide_note", message="refused", field_path="") + assert plan_fix_for_pipe_validation_error(error_data) is None diff --git a/tests/unit/pipelex/pipeline/test_refusal_verdict_arms.py b/tests/unit/pipelex/pipeline/test_refusal_verdict_arms.py new file mode 100644 index 0000000000..3c8c03a060 --- /dev/null +++ b/tests/unit/pipelex/pipeline/test_refusal_verdict_arms.py @@ -0,0 +1,129 @@ +from typing import ClassVar + +import pytest + +from pipelex.base_exceptions import ErrorDomain, PipelexError, SecurityError, ValidationErrorCategory, ValidationErrorItem +from pipelex.cogt.model_backends.model_type import ModelType +from pipelex.core.pipes.exceptions import PipeLoadRefusalError, PipeOperatorModelChoiceError +from pipelex.pipeline.exceptions import ValidateBundleError +from pipelex.pipeline.validate_bundle import translate_to_validate_bundle_error +from pipelex.validation_error_types import PipeValidationErrorType + + +class _CallerFacingInputRefusalError(PipelexError): + error_domain = ErrorDomain.INPUT + _authors_caller_facing_message = True + + +class _InternalInputRefusalError(PipelexError): + error_domain = ErrorDomain.INPUT + _declared_title: ClassVar[str | None] = "Harbour board refusal" + + +class _RuntimeFaultError(PipelexError): + error_domain = ErrorDomain.RUNTIME + + +class _UnclassifiedFaultError(PipelexError): + pass + + +class _InputSecurityRefusalError(SecurityError): + error_domain = ErrorDomain.INPUT + _authors_caller_facing_message = True + + +def _verdict_items(refusal: BaseException) -> tuple[ValidateBundleError, list[ValidationErrorItem]]: + with pytest.raises(ValidateBundleError) as raised, translate_to_validate_bundle_error(): + raise refusal + verdict = raised.value + assert verdict.__cause__ is refusal + return verdict, list(verdict.to_error_report().validation_errors or []) + + +class TestRefusalVerdictArms: + def test_unlocated_caller_facing_input_refusal_is_one_blueprint_item_keeping_its_message(self) -> None: + verdict, items = _verdict_items(_CallerFacingInputRefusalError("The tide table names a harbour that does not exist.")) + + assert items == [ + ValidationErrorItem( + category=ValidationErrorCategory.BLUEPRINT_VALIDATION, + message="The tide table names a harbour that does not exist.", + ) + ] + assert verdict.message == "The tide table names a harbour that does not exist." + + def test_non_caller_facing_input_refusal_is_named_by_its_title(self) -> None: + _, items = _verdict_items(_InternalInputRefusalError("internal detail: registry slot 7 is stale")) + + assert [item.message for item in items] == ["Harbour board refusal"] + + def test_located_refusal_is_one_pipe_item_with_pipe_domain_and_source(self) -> None: + cause = _InternalInputRefusalError("internal detail: registry slot 7 is stale") + located = PipeLoadRefusalError.make_from_refusal(refusal=cause, pipe_code="write_tide_note", domain_code="tide_tables", source="tides.mthds") + + _, items = _verdict_items(located) + + assert items == [ + ValidationErrorItem( + category=ValidationErrorCategory.PIPE_VALIDATION, + pipe_code="write_tide_note", + domain_code="tide_tables", + source="tides.mthds", + message="Pipe 'write_tide_note' could not be loaded: Harbour board refusal", + ) + ] + + def test_unknown_model_is_one_unknown_model_item(self) -> None: + refusal = PipeOperatorModelChoiceError( + "Pipe 'write_tide_note' (PipeLLM), field 'model': Model handle 'gpt-5.1' was not found in the model deck", + pipe_type="PipeLLM", + pipe_code="write_tide_note", + domain_code="tide_tables", + field_name="model", + model_type=ModelType.LLM, + model_choice="gpt-5.1", + suggestions=["gpt-5.5", "gpt-5.4"], + source="tides.mthds", + ) + + _, items = _verdict_items(refusal) + + assert items == [ + ValidationErrorItem( + category=ValidationErrorCategory.PIPE_VALIDATION, + error_type=PipeValidationErrorType.UNKNOWN_MODEL, + pipe_code="write_tide_note", + domain_code="tide_tables", + source="tides.mthds", + field_path="pipe.write_tide_note.model", + field_name="model", + model_reference="gpt-5.1", + model_type="llm", + suggestions=["gpt-5.5", "gpt-5.4"], + message="Pipe 'write_tide_note' (PipeLLM), field 'model': Model handle 'gpt-5.1' was not found in the model deck", + ) + ] + + @pytest.mark.parametrize( + "fault", + [ + _RuntimeFaultError("the worker lost its connection"), + _UnclassifiedFaultError("something nobody classified"), + _InputSecurityRefusalError("a fetch was blocked"), + ], + ids=["runtime", "unclassified", "security"], + ) + def test_no_verdict_faults_propagate_unchanged(self, fault: PipelexError) -> None: + """A failure of the tool or its environment, and a security refusal, are never reported as the bundle's fault.""" + with pytest.raises(type(fault)) as raised, translate_to_validate_bundle_error(): + raise fault + assert raised.value is fault + + def test_a_produced_verdict_passes_through_unwrapped(self) -> None: + verdict = ValidateBundleError(message="already a verdict", dry_run_error_message="the round has no lane") + + with pytest.raises(ValidateBundleError) as raised, translate_to_validate_bundle_error(): + raise verdict + + assert raised.value is verdict From d570f4a7d950477d4e4410765cf2197e9a109a3c Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 26 Sep 2026 18:47:22 +0200 Subject: [PATCH 2/5] Review fixes: unsafe rename-model, accurate validate docs The rename-model fix is now UNSAFE: its one suggestion is a fuzzy match over the deck's names, which can be a different model, so `pipelex fix bundle` never applies it on its own. The availability handler's docstring again says validate passes exit code 2 for a model no backend serves. The CLI docs now say that agent `validate pipe` answers a failing dry run with the DryRunError envelope, and that a refusal whose class does not declare the input domain still leaves as no verdict. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FigDssaJrvNcmbnBedi7oq --- .drift/acks/cli-docs.toml | 8 ++++---- CHANGELOG.md | 2 +- docs/tools/cli/agent-cli.md | 2 +- docs/tools/cli/validate.md | 4 ++-- docs/under-the-hood/error-model.md | 4 ++-- pipelex/cli/error_handlers.py | 6 +++--- pipelex/pipeline/fixes/planner.py | 9 +++++++-- .../pipeline/test_validate_bundle_load_refusals.py | 2 +- tests/unit/pipelex/pipeline/fixes/test_fix_planner.py | 2 +- 9 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml index 6fc2adb012..980378baee 100644 --- a/.drift/acks/cli-docs.toml +++ b/.drift/acks/cli-docs.toml @@ -1,8 +1,8 @@ contract = "cli-docs" -digest = "sha256:f03ae0caaedf4e4b644af61e7ace8a4d9c08597e9b1a48906a9334daa8e5a27a" +digest = "sha256:7e3a9c06b9fcecfc2dd24e2a1553c2d2cfe31b95c455d2911e857163874f1e86" reviewed_by = "Louis Choquel" -reviewed_at = "2026-09-26T16:11:26Z" -rationale = "The validate commands (bare and agent) now run their library load inside translate_to_validate_bundle_error, so a load-time refusal and the unknown model leave as a ValidateBundleError verdict with exit 1; the PipeOperatorModelChoiceError exit-2 arms and its config entry in AGENT_ERROR_DOMAINS are gone (the class declares its input domain). Reviewed docs/tools/cli/validate.md and agent-cli.md (updated with the verdict sections) and pipelex/cli/agent_cli/CLAUDE.md (new Validate refusals are verdicts bullet)." +reviewed_at = "2026-09-26T16:40:03Z" +rationale = "Review fix: the handle_model_availability_error docstring again says validate passes exit code 2 for a model no backend serves; docs/tools/cli/agent-cli.md and validate.md now state that agent validate pipe answers a failing dry run with the DryRunError envelope and that refusals not classified as input still leave as no verdict, and the rename-model fix is described as unsafe." [trigger_files] "pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc" @@ -127,7 +127,7 @@ rationale = "The validate commands (bare and agent) now run their library load i "pipelex/cli/commands/validate/pipe_cmd.py" = "blob:d67267571559006459628180f2c8b216ba7fee31" "pipelex/cli/commands/which_cmd.py" = "blob:68661d4bab22e2c4dab183d59bd9f2c6dd573472" "pipelex/cli/deck_notice.py" = "blob:542877125e89e35b02bc969cfee952155b0d7c2c" -"pipelex/cli/error_handlers.py" = "blob:b0fb1813633f380a0c4e4f1ea2b2f5b2d57b2c23" +"pipelex/cli/error_handlers.py" = "blob:c704505651f24a0e9bb85fed0da1d00445c9e980" "pipelex/cli/exceptions.py" = "blob:1d51c3a53ddd3f6cf8de566fa2ade3fa75ab7a3a" "pipelex/cli/installed_methods.py" = "blob:09197f47baf8d7251d28d148a0544b4a23ee7365" "pipelex/cli/method_resolver.py" = "blob:1230dde5f89ac28bbdcbcbbd0c1d485c71803caf" diff --git a/CHANGELOG.md b/CHANGELOG.md index 988b5938ce..8e93825852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **An unknown model is a located `unknown_model` validation item**: a pipe whose model field names a handle, alias, preset or waterfall the model deck does not define now validates to an invalid verdict with one `pipe_validation` item of the new closed `error_type` `unknown_model`, carrying the `pipe_code`, `domain_code`, `source`, `field_name`, the `field_path` of the reference (`pipe..model`, or `pipe..model_to_structure`), and the new optional item fields `model_reference` (the reference as written), `model_type` and `suggestions` (the deck's close matches of the same kind, also kept in the message). Every pipe type that names a model gives the same item, where `PipeExtract` and `PipeSearch` used to give an `unknown_validation_error` whose message was a Python repr, and with exactly one suggestion the item carries a safe `rename-model` fix that `pipelex fix bundle` applies. The MTHDS Test Corpus gains the `invalid_unknown_model` entry covering the new `error.unknown_model` tag. +- **An unknown model is a located `unknown_model` validation item**: a pipe whose model field names a handle, alias, preset or waterfall the model deck does not define now validates to an invalid verdict with one `pipe_validation` item of the new closed `error_type` `unknown_model`, carrying the `pipe_code`, `domain_code`, `source`, `field_name`, the `field_path` of the reference (`pipe..model`, or `pipe..model_to_structure`), and the new optional item fields `model_reference` (the reference as written), `model_type` and `suggestions` (the deck's close matches of the same kind, also kept in the message). Every pipe type that names a model gives the same item, where `PipeExtract` and `PipeSearch` used to give an `unknown_validation_error` whose message was a Python repr, and with exactly one suggestion the item carries an unsafe `rename-model` fix, which `pipelex fix bundle` never applies on its own because a fuzzy match can name a different model. The MTHDS Test Corpus gains the `invalid_unknown_model` entry covering the new `error.unknown_model` tag. ### Changed diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.md index 0fa8a48105..86ff0249bb 100644 --- a/docs/tools/cli/agent-cli.md +++ b/docs/tools/cli/agent-cli.md @@ -80,7 +80,7 @@ For `bundle`, additional options are available: On a successful run, the envelope also carries `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), each namespaced by `pipe_ref` (`domain.code`). In JSON it is a `pending_signatures` array, in markdown a "Pending signatures" section. A top-down build reads it to see exactly which headers remain to implement. The envelope also carries a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown states the runnability verdict in plain English — runnable when complete, NOT yet runnable above the "Pending signatures" section otherwise. `validate bundle`, `validate method`, and `validate pipe --all` carry them and gate on them: without `--allow-signatures`, the command exits non-zero when `is_runnable` is false. Bare `validate pipe ` omits them, and a `--pipe` slice surfaces them for information without gating. !!! note "Every refusal of the bundle is an invalid verdict" - A refusal raised while the bundle is loaded or validated is answered with the invalid-verdict envelope — `is_valid: false` and a `validation_errors` array — and exit code `1`, never the no-verdict envelope and exit code `2`, which is kept for failures of the tool or its environment. A pipe naming a model the model deck does not define gives one `pipe_validation` item with `error_type: unknown_model`, carrying `pipe_code`, `domain_code`, `source`, `field_path` (`pipe..model`), `model_reference` (the reference as written), `model_type` and `suggestions` (the deck's close matches), plus a `rename-model` `suggested_fix` when there is exactly one suggestion. Any other refusal of your input with no code of its own gives one item without an `error_type`, located on its pipe when it was raised while that pipe was built. See [Error Model](../../under-the-hood/error-model.md#validation_errors-structured-bundle-validation-diagnostics). + A refusal of your input raised while the bundle is loaded or validated is answered with the invalid-verdict envelope — `is_valid: false` and a `validation_errors` array — and exit code `1`, never the no-verdict envelope and exit code `2`, which is kept for failures of the tool or its environment. `validate pipe ` and `validate pipe --all` are the exception for a failing dry run: they answer it with `is_valid: false`, `error_type: DryRunError` and exit code `1`, without a `validation_errors` array. A pipe naming a model the model deck does not define gives one `pipe_validation` item with `error_type: unknown_model`, carrying `pipe_code`, `domain_code`, `source`, `field_path` (`pipe..model`), `model_reference` (the reference as written), `model_type` and `suggestions` (the deck's close matches), plus a `rename-model` `suggested_fix` when there is exactly one suggestion. Any other refusal of your input with no code of its own gives one item without an `error_type`, located on its pipe when it was raised while that pipe was built. A refusal whose class does not classify itself as your input yet, as some pipe-factory refusals do not, still leaves as no verdict. See [Error Model](../../under-the-hood/error-model.md#validation_errors-structured-bundle-validation-diagnostics). !!! note "Advisory warnings on validate" Whole-bundle and whole-library validate surfaces (`validate bundle`, `validate method`, `validate pipe --all`) also carry a `warnings` array — advisory optionality lints on a VALID bundle that never flip the verdict or the exit code. Each entry has the **same shape as a validation error item** (`category`, `error_type`, `pipe_code`, `domain_code`, `variable_names`, `message`) — this is a different shape from the `init`/`doctor` setup `warnings` (`{type, message}`) documented under Output Contract below. Three families ride the array, always in this order: the useless-`!` lint (`optional_force_redundant`), a `!` (force) input whose slot is guaranteed present in every analyzed flow, so the assertion can never fire; the vacuous-presence lint (`input_presence_vacuous`), an entry-pipe input that must be supplied but whose concept declares no required field, so the empty object satisfies it and a caller cannot tell what to fill in (see [Understanding Optionality](../../building-methods/pipes/understanding-optionality.md)); and the [intent-hint](../../building-methods/concepts/intent-hints.md) lints (`hint_unknown_key`, `hint_unknown_intent`, `hint_inapplicable_intent`). Every whole-bundle validate channel — this CLI, the bare CLI, the builder ops and the protocol validation report — assembles them from one composition point, so which advisories you see does not depend on which command you typed. Hint findings are bounded per site: a site naming many undefined keys reports the first few and then how many more there were, and a long authored key or value is elided in the message. In markdown, warnings render as a "Warnings" section. The array is empty when there is nothing to report; `validate pipe` omits it (no flow context to lint in). `validate bundle`/`validate method` with `--pipe` keep it, and it stays bundle-wide there — the slice narrows the dry run, not the validation. diff --git a/docs/tools/cli/validate.md b/docs/tools/cli/validate.md index 51d23d6040..d7522406ca 100644 --- a/docs/tools/cli/validate.md +++ b/docs/tools/cli/validate.md @@ -124,7 +124,7 @@ pipelex validate method invoice_extractor --pipe extract_amounts ## An Invalid Bundle Is a Verdict -Every refusal of your method raised while it is loaded or validated is reported as an invalid bundle: the grouped `❌ Bundle validation failed` output naming each error's pipe, domain, field and file, with exit code `1`. Exit code `2` is kept for the cases where no verdict could be produced — bad arguments, a target that does not resolve, or a failure of Pipelex or its environment. This holds on every subcommand: `validate pipe` and `validate --all` render a pipe whose dry run fails the same way `validate bundle` does, where they used to print a traceback. +Every refusal of your method raised while it is loaded or validated is reported as an invalid bundle: the grouped `❌ Bundle validation failed` output naming each error's pipe, domain, field and file, with exit code `1`. Exit code `2` is kept for the cases where no verdict could be produced — bad arguments, a target that does not resolve, or a failure of Pipelex or its environment. This holds on every subcommand: `validate pipe` and `validate --all` render a pipe whose dry run fails the same way `validate bundle` does, where they used to print a traceback. A few refusals raised while a pipe is built do not classify themselves as a fault of your method yet, and those still stop validation without a verdict. A pipe that names a model your model deck does not define is the most common case. It is reported as an `Unknown Model` error on that pipe, with the path of the field that names it (`pipe..model`, or `pipe..model_to_structure` for the model a `PipeLLM` structures its output with) and the deck's close matches: @@ -142,7 +142,7 @@ Did you mean: @best-gpt └─ Path: pipe.write_tide_note.model ``` -When the deck offers exactly one close match, as here, the error carries a suggested fix that `pipelex fix bundle` applies. With several, choosing among them is yours; `pipelex-agent check-model -t ` and `pipelex-agent models -t ` list what the deck defines. +When the deck offers exactly one close match, as here, the error carries a suggested fix naming it. The fix is marked unsafe and `pipelex fix bundle` does not apply it on its own: the match is a guess from the spelling, and a close name can be a different model, with its own provider, cost and behaviour, so check it before you write it. With several matches, choosing among them is yours; `pipelex-agent check-model -t ` and `pipelex-agent models -t ` list what the deck defines. ## Suggested Fixes diff --git a/docs/under-the-hood/error-model.md b/docs/under-the-hood/error-model.md index 3a0490349e..d15722429a 100644 --- a/docs/under-the-hood/error-model.md +++ b/docs/under-the-hood/error-model.md @@ -85,9 +85,9 @@ Together the two residuals make the **structured-info invariant total**: every i Besides `category` and `message`, each item carries whatever identity fields its stage produced — `error_type`, `pipe_code`, `concept_code`, `domain_code`, `field_path`, `field_name`, `variable_names`, `missing_concept_code`, `declared_concepts`, the unknown-model locators `model_reference`, `model_type` and `suggestions`, and a `source` (the declaring file path, or the per-content source the in-memory load path was given) that hands a consumer the owning file for cross-file diagnostic placement. When the error has a deterministic remedy, the item also carries a [`suggested_fix`](#suggested_fix-structured-deterministic-fixes). -**Every refusal of the bundle is a verdict.** A validator answers either a verdict — valid, or invalid with located items — or *no verdict could be produced*, which is reserved for a failure of the tool or its environment. So a refusal raised while loading or validating a bundle becomes an item, never a no-verdict fault. After its class-specific arms, the shared cascade (`translate_to_validate_bundle_error` in `pipelex/pipeline/validate_bundle.py`) turns any other `PipelexError` whose report is `input`-domained into a verdict with one item: `pipe_validation`, with the `pipe_code`, `domain_code` and `source`, when the library load located it on the pipe it was building, and `blueprint_validation` otherwise. That item carries no `error_type`, since a refusal with a closed code has an arm of its own. It keeps the refusal's message only when the refusal authored that message as caller-facing copy, and otherwise names the refusal's title, because the verdict as a whole is kept verbatim under STRICT disclosure. The location comes from the load loop in `LibraryManager.load_from_crate`, the one place that holds a pipe's code, domain and file when building it fails: it raises such a refusal again as a `PipeLoadRefusalError` naming the pipe and the file, `from` the original. A `config` or `runtime` fault, an unclassified one, a `SecurityError`, a `PipeNotFoundError` (which has its own not-found handler) and anything that is not a `PipelexError` still propagate as no verdict. +**Every refusal of the bundle is a verdict.** A validator answers either a verdict — valid, or invalid with located items — or *no verdict could be produced*, which is reserved for a failure of the tool or its environment. So a refusal raised while loading or validating a bundle becomes an item, never a no-verdict fault. After its class-specific arms, the shared cascade (`translate_to_validate_bundle_error` in `pipelex/pipeline/validate_bundle.py`) turns any other `PipelexError` whose report is `input`-domained into a verdict with one item: `pipe_validation`, with the `pipe_code`, `domain_code` and `source`, when the library load located it on the pipe it was building, and `blueprint_validation` otherwise. That item carries no `error_type`, since a refusal with a closed code has an arm of its own. It keeps the refusal's message only when the refusal authored that message as caller-facing copy, and otherwise names the refusal's title, because the verdict as a whole is kept verbatim under STRICT disclosure. The location comes from the load loop in `LibraryManager.load_from_crate`, the one place that holds a pipe's code, domain and file when building it fails: it raises such a refusal again as a `PipeLoadRefusalError` naming the pipe and the file, `from` the original. A `config` or `runtime` fault, an unclassified one, a `SecurityError`, a `PipeNotFoundError` (which has its own not-found handler) and anything that is not a `PipelexError` still propagate as no verdict. The rule therefore reaches exactly the refusals whose class declares the `input` domain: an authoring refusal that declares none yet, such as `PipeExtractFactoryError` for a `PipeExtract` whose input is neither an image nor a document, still propagates as no verdict until its class is classified. -**The unknown model is the worked example.** A pipe whose model field names a handle, an alias, a preset or a waterfall the model deck does not define is refused when the pipe is built: the operator's deck check raises `ModelChoiceNotFoundError`, and the operator raises it again as a `PipeOperatorModelChoiceError` located on the pipe and on the field, to which the load adds the file. Every pipe type that names a model (`PipeLLM`, `PipeStructure`, `PipeImgGen`, `PipeExtract`, `PipeSearch`) does this through the same `PipeOperator.locating_model_choice` wrapper, so each gives the same item: `category: pipe_validation`, `error_type: unknown_model`, the `pipe_code`, `domain_code` and `source`, the `field_name` and a `field_path` of `pipe..model` (`pipe..model_to_structure` for a `PipeLLM`'s structuring model), the `model_reference` exactly as written, the `model_type` (`llm`, `text_extractor`, `img_gen` or `search`), and the deck's `suggestions` of the same kind, each spelled as a reference the field accepts. The suggestions stay in the item's `message` too, for consumers that keep only the message. When the deck offers exactly one suggestion, the item carries a safe `rename-model` fix, a `remap_value` of the field from the reference as written to that suggestion. An inline setting table (`model = { model = "…", temperature = 0.2 }`) is not looked up in the deck, so it is not refused here. On a run the same `PipeOperatorModelChoiceError` refuses the bundle before any pipe runs: it is `input`-domained and caller-facing, so the hosted answer is a 422 whose message names the pipe and keeps the deck check's sentence. +**The unknown model is the worked example.** A pipe whose model field names a handle, an alias, a preset or a waterfall the model deck does not define is refused when the pipe is built: the operator's deck check raises `ModelChoiceNotFoundError`, and the operator raises it again as a `PipeOperatorModelChoiceError` located on the pipe and on the field, to which the load adds the file. Every pipe type that names a model (`PipeLLM`, `PipeStructure`, `PipeImgGen`, `PipeExtract`, `PipeSearch`) does this through the same `PipeOperator.locating_model_choice` wrapper, so each gives the same item: `category: pipe_validation`, `error_type: unknown_model`, the `pipe_code`, `domain_code` and `source`, the `field_name` and a `field_path` of `pipe..model` (`pipe..model_to_structure` for a `PipeLLM`'s structuring model), the `model_reference` exactly as written, the `model_type` (`llm`, `text_extractor`, `img_gen` or `search`), and the deck's `suggestions` of the same kind, each spelled as a reference the field accepts. The suggestions stay in the item's `message` too, for consumers that keep only the message. When the deck offers exactly one suggestion, the item carries an `unsafe` `rename-model` fix, a `remap_value` of the field from the reference as written to that suggestion. It is unsafe because the suggestion is a fuzzy match over the deck's names, which can be a different model altogether, so `pipelex fix bundle` never applies it on its own; an author or an agent applies it deliberately. An inline setting table (`model = { model = "…", temperature = 0.2 }`) is not looked up in the deck, so it is not refused here. On a run the same `PipeOperatorModelChoiceError` refuses the bundle before any pipe runs: it is `input`-domained and caller-facing, so the hosted answer is a 422 whose message names the pipe and keeps the deck check's sentence. **Signatures are never an error.** An unimplemented `PipeSignature` reached during validation is a *runnability fact*, not a validation failure: the validator no longer raises on it. The assembled library's outstanding signatures ride the validation report's `pending_signatures`, and `is_runnable = not pending_signatures`. `allow_signatures` is a sweep-mechanics flag only (whether signature pipes are mock-run and listed in `validated_pipes`) — it does not change the verdict, so strict ≡ lenient in the report body. The "is this a failure?" decision moves to the consumer: the CLI exits non-zero on `not is_runnable` unless `--allow-signatures`; the HTTP caller reads `is_runnable`. (The **execute/run** path is different: running a stub still raises `PipeSignatureNotExecutableError`.) diff --git a/pipelex/cli/error_handlers.py b/pipelex/cli/error_handlers.py index b0fb181363..c704505651 100644 --- a/pipelex/cli/error_handlers.py +++ b/pipelex/cli/error_handlers.py @@ -158,9 +158,9 @@ def handle_model_availability_error(exc: PipeOperatorModelAvailabilityError, *, Args: exc: The model availability error exception context: Context for the error message - exit_code: Process exit code; the default is 1. The validate surface never reaches - this handler: an unknown model is an invalid verdict there, rendered by - :func:`handle_validate_bundle_error`. + exit_code: Process exit code. The validate surface passes 2: a model the deck defines + but no enabled backend serves is a setup fault of this machine, so no verdict, unlike + an unknown model, which is an invalid verdict there. Other contexts keep the default 1. """ console = get_console() print_traceback_if_requested(console=console) diff --git a/pipelex/pipeline/fixes/planner.py b/pipelex/pipeline/fixes/planner.py index adc9e88454..71a1fb787e 100644 --- a/pipelex/pipeline/fixes/planner.py +++ b/pipelex/pipeline/fixes/planner.py @@ -50,10 +50,15 @@ def _plan_rename_model(*, error_data: PipesAndConceptValidationErrorData) -> Sug """``rename-model``: an ``unknown_model`` whose deck offers exactly one close match of the same kind becomes a ``remap_value`` of the pipe's model field, from the reference as written to that match. - A single match is the only case with one obvious correction; with several, choosing among them is + A single match is the only case with one candidate correction; with several, choosing among them is the author's call, and with none there is nothing to write. The remap names the reference as written, so the op rewrites the field only while it still holds that exact value and leaves a field the author has since edited alone. + + The fix is ``UNSAFE``: the match is a fuzzy guess (a similarity cutoff over the deck's names), so a + single match can still be a different model, with its own provider, cost and behaviour, and a wrong + sigil the author meant is not weighed against it. It rides the item for an author or an agent to + apply deliberately, and ``pipelex fix bundle`` never applies it on its own. """ if error_data.pipe_code is None or error_data.field_name is None or error_data.model_reference is None: return None @@ -64,7 +69,7 @@ def _plan_rename_model(*, error_data: PipesAndConceptValidationErrorData) -> Sug return SuggestedFix( fix_code=RENAME_MODEL_FIX_CODE, description=f"{description}, its one close match in the model deck", - safety=FixSafety.SAFE, + safety=FixSafety.UNSAFE, source=error_data.source, ops=[ RemapValueOp( diff --git a/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py index a257c18f3e..8c3cf997d5 100644 --- a/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py +++ b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py @@ -242,7 +242,7 @@ async def test_one_suggestion_carries_a_rename_fix(self, tmp_path: Path) -> None fix = item.suggested_fix assert fix is not None assert fix.fix_code == RENAME_MODEL_FIX_CODE - assert fix.safety.is_safe + assert not fix.safety.is_safe, "a fuzzy match is never auto-applied" assert fix.source == str(bundle_path) assert fix.ops == [RemapValueOp(table_path=["pipe", "write_tide_note"], key="model", mapping={"@best-sonet": "@best-gpt"})] diff --git a/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py b/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py index 273ad8e3c2..0994db8ca3 100644 --- a/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py +++ b/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py @@ -215,7 +215,7 @@ def test_unknown_model_with_one_suggestion_yields_rename_model_fix(self) -> None fix = plan_fix_for_pipe_validation_error(_unknown_model_error_data(suggestions=["@best-gpt"])) assert fix is not None assert fix.fix_code == RENAME_MODEL_FIX_CODE - assert fix.safety == FixSafety.SAFE + assert fix.safety == FixSafety.UNSAFE assert fix.source == "main.mthds" assert fix.ops == [RemapValueOp(table_path=["pipe", "write_tide_note"], key="model", mapping={"@best-sonet": "@best-gpt"})] assert "'@best-sonet'" in fix.description From ee2a7bb933b61dc8de167235d979f1e8c16008fd Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 26 Sep 2026 19:25:08 +0200 Subject: [PATCH 3/5] Review round 2: locate helper and dependency refusals, unsafe codes unselectable A refusal raised while building a synthetic helper of a preliminary_text PipeLLM is now located on the authored pipe and field, because the library crate carries the elaborator's metadata beside its source map; before, the unknown_model item named `__structure` and its fix targeted a table the author never wrote. Pipes of a dependency package are built under the same locating context as the main load, so their refusals name their file. The unsafe rename-model code is no longer accepted by --select/--ignore, and the "suggested fix not applied" tip counts only safe fixes. Co-Authored-By: Claude Opus 5.5 --- .drift/acks/cli-docs.toml | 8 +-- docs/tools/cli/fix.md | 2 +- pipelex/cli/commands/fix/_fix_core.py | 4 +- pipelex/libraries/crate_normalization.py | 1 + pipelex/libraries/library_crate.py | 7 +- pipelex/libraries/library_crate_factory.py | 19 ++++- pipelex/libraries/library_manager.py | 72 ++++++++++++++++--- pipelex/pipeline/fixes/fix_render.py | 4 +- pipelex/pipeline/fixes/planner.py | 4 +- .../test_validate_bundle_load_refusals.py | 48 +++++++++++++ .../pipeline/fixes/test_fix_planner.py | 3 +- 11 files changed, 149 insertions(+), 23 deletions(-) diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml index 980378baee..cf9e238932 100644 --- a/.drift/acks/cli-docs.toml +++ b/.drift/acks/cli-docs.toml @@ -1,8 +1,8 @@ contract = "cli-docs" -digest = "sha256:7e3a9c06b9fcecfc2dd24e2a1553c2d2cfe31b95c455d2911e857163874f1e86" +digest = "sha256:23181f87f119d19610be5c07c0757df6d1c4fdbca32c2265a9d0987053cb4790" reviewed_by = "Louis Choquel" -reviewed_at = "2026-09-26T16:40:03Z" -rationale = "Review fix: the handle_model_availability_error docstring again says validate passes exit code 2 for a model no backend serves; docs/tools/cli/agent-cli.md and validate.md now state that agent validate pipe answers a failing dry run with the DryRunError envelope and that refusals not classified as input still leave as no verdict, and the rename-model fix is described as unsafe." +reviewed_at = "2026-09-26T17:15:21Z" +rationale = "Review round 2: the fix commands' 'suggested fix not applied' tip now counts only safe fixes, and rename-model (unsafe) is not a --select/--ignore code; docs/tools/cli/fix.md says so. validate.md and agent-cli.md reviewed, no further change needed." [trigger_files] "pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc" @@ -85,7 +85,7 @@ rationale = "Review fix: the handle_model_availability_error docstring again say "pipelex/cli/commands/doctor_cmd.py" = "blob:26db32ff20d1c064bc421093825a05e980c9b770" "pipelex/cli/commands/fix/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" "pipelex/cli/commands/fix/_diff_sandbox.py" = "blob:5ce264f5b0ab4ff4172f4d8789450916947f5c0e" -"pipelex/cli/commands/fix/_fix_core.py" = "blob:8f6a187a0fb42c498629b177753d81620800b0da" +"pipelex/cli/commands/fix/_fix_core.py" = "blob:dac7056fcaf50210c910e4cad1ba14328b7f8adc" "pipelex/cli/commands/fix/app.py" = "blob:6f019930e8e3e0043ee46d87493ba4602eb33a76" "pipelex/cli/commands/fix/bundle_cmd.py" = "blob:427364cc453957dfe0cec879f019050329c13b3c" "pipelex/cli/commands/graph_cmd.py" = "blob:d54737af7efc9ab9bbf3f978050d017af0e27a41" diff --git a/docs/tools/cli/fix.md b/docs/tools/cli/fix.md index ee000c4fa5..076e4b23e3 100644 --- a/docs/tools/cli/fix.md +++ b/docs/tools/cli/fix.md @@ -60,7 +60,7 @@ pipelex fix bundle my_bundle.mthds --select match-sequence-output ## What Gets Fixed -Only fixes classified as SAFE are applied — deterministic corrections derived from the structured validation errors, never from guesswork. Errors without a safe deterministic fix are left in place and reported as remaining errors. The available fix rule codes (for `--select`/`--ignore`) are listed in the error message when you pass an unknown code. +Only fixes classified as SAFE are applied — deterministic corrections derived from the structured validation errors, never from guesswork. Errors without a safe deterministic fix are left in place and reported as remaining errors. An unsafe fix, such as `rename-model` for a model name the deck only nearly matches, still shows on its error for you to apply by hand, and is not a code `--select` or `--ignore` accepts. The available fix rule codes (for `--select`/`--ignore`) are listed in the error message when you pass an unknown code. ## Related Documentation diff --git a/pipelex/cli/commands/fix/_fix_core.py b/pipelex/cli/commands/fix/_fix_core.py index 8f6a187a0f..dac7056fca 100644 --- a/pipelex/cli/commands/fix/_fix_core.py +++ b/pipelex/cli/commands/fix/_fix_core.py @@ -124,10 +124,10 @@ def _render_fix_result(*, console: Console, result: FixBundleResult, bundle_path console.print("[bold cyan]Remaining errors:[/bold cyan]\n") display_validation_error_items(console=console, items=result.remaining_errors) - # A remaining error can still carry a 💡 suggested-fix line — dropped by --select/--ignore, + # A remaining error can still carry a safe 💡 suggested-fix line — dropped by --select/--ignore, # left outside the write scope, or unconverged when the loop bailed. Claiming "no safe fix" # there contradicts the line just printed, so only say it when nothing fixable remains. - if any(item.suggested_fix is not None for item in result.remaining_errors): + if any(item.suggested_fix is not None and item.suggested_fix.safety.is_safe for item in result.remaining_errors): console.print( "[bold green]💡 Tip:[/bold green] Some remaining errors above still show a suggested fix that was not applied — " "they were skipped by --select/--ignore, fell outside the write scope, or the loop stopped early (see the reason above). " diff --git a/pipelex/libraries/crate_normalization.py b/pipelex/libraries/crate_normalization.py index f3a47dd4d5..100aeeadab 100644 --- a/pipelex/libraries/crate_normalization.py +++ b/pipelex/libraries/crate_normalization.py @@ -99,6 +99,7 @@ def normalize_crate(crate: LibraryCrate, *, mthds_version: str) -> LibraryCrate: pipes=pipes, domains=crate.domains, source_map=crate.source_map, + elaboration_metadata=crate.elaboration_metadata, fingerprint=fingerprint, ) diff --git a/pipelex/libraries/library_crate.py b/pipelex/libraries/library_crate.py index 5f50c186a5..bde4ab2d23 100644 --- a/pipelex/libraries/library_crate.py +++ b/pipelex/libraries/library_crate.py @@ -5,7 +5,7 @@ from pipelex.core.concepts.concept_blueprint import ConceptBlueprint from pipelex.core.domains.domain_blueprint import DomainBlueprint -from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipeBlueprintUnion +from pipelex.mthds_parsing.pipelex_bundle_blueprint import ElaborationMetadata, PipeBlueprintUnion class LibraryCrate(BaseModel): @@ -35,6 +35,11 @@ class LibraryCrate(BaseModel): source_map: dict[str, str] = Field(default_factory=dict) """concept_ref or pipe_ref -> source file path (for error reporting)""" + elaboration_metadata: dict[str, ElaborationMetadata] = Field(default_factory=dict) + """pipe_ref -> how the bundle elaborator generated that synthetic pipe (the helpers of a + ``preliminary_text`` PipeLLM), so a refusal raised while building a helper is reported on the pipe + and the field the author wrote. Like ``source_map``, it is excluded from the fingerprint.""" + python_sources: dict[str, str] = Field(default_factory=dict) """relpath (within the library dir) -> Python source text, captured WITHOUT importing. diff --git a/pipelex/libraries/library_crate_factory.py b/pipelex/libraries/library_crate_factory.py index b69b216d07..f4bc74986e 100644 --- a/pipelex/libraries/library_crate_factory.py +++ b/pipelex/libraries/library_crate_factory.py @@ -8,7 +8,7 @@ from pipelex.libraries.domain.domain_metadata_merge import merge_domain_metadata_field from pipelex.libraries.library_crate import LibraryCrate from pipelex.libraries.pipe.exceptions import PipeLibraryError -from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipeBlueprintUnion, PipelexBundleBlueprint +from pipelex.mthds_parsing.pipelex_bundle_blueprint import ElaborationMetadata, PipeBlueprintUnion, PipelexBundleBlueprint from pipelex.pipe_machinery.pipe_factory import PipeFactory if TYPE_CHECKING: @@ -60,6 +60,7 @@ def make_from_blueprints( pipes: dict[str, PipeBlueprintUnion] = {} domains: dict[str, DomainBlueprint] = {} source_map: dict[str, str] = {} + elaboration_metadata: dict[str, ElaborationMetadata] = {} for blueprint in blueprints: domain_code = blueprint.domain @@ -141,10 +142,17 @@ def make_from_blueprints( source_map[pipe_ref] = winner.source else: source_map.pop(pipe_ref, None) + if winner.blueprint is pipe_blueprint: + cls._track_elaboration( + elaboration_metadata=elaboration_metadata, pipe_ref=pipe_ref, elaboration=blueprint.get_elaboration_for(pipe_code) + ) continue pipes[pipe_ref] = pipe_blueprint if source: source_map[pipe_ref] = source + cls._track_elaboration( + elaboration_metadata=elaboration_metadata, pipe_ref=pipe_ref, elaboration=blueprint.get_elaboration_for(pipe_code) + ) # Concept-reference validation is NOT done here: this factory performs a world-agnostic # structural merge. Same-domain concept references resolve against the live library (which @@ -156,10 +164,19 @@ def make_from_blueprints( pipes=pipes, domains=domains, source_map=source_map, + elaboration_metadata=elaboration_metadata, python_sources=python_sources or {}, fingerprint=fingerprint, ) + @staticmethod + def _track_elaboration(*, elaboration_metadata: dict[str, ElaborationMetadata], pipe_ref: str, elaboration: ElaborationMetadata | None) -> None: + """Keep the elaboration side-table on the declaration that won ``pipe_ref``, like ``source_map``.""" + if elaboration is None: + elaboration_metadata.pop(pipe_ref, None) + else: + elaboration_metadata[pipe_ref] = elaboration + @classmethod def _reconcile_pipe_collision( cls, diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py index a43a24f80c..6520af2479 100644 --- a/pipelex/libraries/library_manager.py +++ b/pipelex/libraries/library_manager.py @@ -17,6 +17,7 @@ import pipelex.builder as builder_pkg # package import — used for __file__ path from pipelex import log from pipelex.base_exceptions import PipelexError, SecurityError, error_domain_is_input +from pipelex.cogt.exceptions import ModelChoiceNotFoundError from pipelex.config import is_pipe_func_sandbox_hosted from pipelex.core.concepts.concept_blueprint import ConceptBlueprint from pipelex.core.concepts.concept_factory import ConceptFactory @@ -50,7 +51,7 @@ from pipelex.mthds_parsing.exceptions import MthdsParserError from pipelex.mthds_parsing.handle_pipe_errors import categorize_pipe_validation_error from pipelex.mthds_parsing.parser import MthdsParser -from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprint +from pipelex.mthds_parsing.pipelex_bundle_blueprint import ElaborationMetadata, PipelexBundleBlueprint, StepRole from pipelex.pipe_machinery.pipe_abstract import PipeAbstract from pipelex.pipe_machinery.pipe_factory import PipeFactory from pipelex.runtime_hub import get_class_registry @@ -102,8 +103,37 @@ def _find_methods_dirs_from_blueprints(blueprints: list[PipelexBundleBlueprint]) return result +def _authored_model_field(*, step_role: StepRole) -> str: + """The field of the authored ``preliminary_text`` PipeLLM whose model a synthetic helper was given.""" + match step_role: + case StepRole.DRAFT_TEXT: + return "model" + case StepRole.STRUCTURE: + return "model_to_structure" + + +def _relocate_on_authored_pipe( + *, model_choice_error: PipeOperatorModelChoiceError, elaboration: ElaborationMetadata, domain_code: str +) -> PipeOperatorModelChoiceError: + """Move an unknown model refused on a synthetic helper onto the authored pipe and field it came from.""" + cause = model_choice_error.__cause__ + if not isinstance(cause, ModelChoiceNotFoundError): + return model_choice_error + relocated = PipeOperatorModelChoiceError.make_from_model_choice_not_found( + model_choice_error=cause, + pipe_type="PipeLLM", + pipe_code=elaboration.parent_pipe_code, + domain_code=domain_code, + field_name=_authored_model_field(step_role=elaboration.step_role), + ) + relocated.source = model_choice_error.source + return relocated + + @contextmanager -def _locating_pipe_build_refusals(*, pipe_code: str, domain_code: str, source: str | None) -> Generator[None, None, None]: +def _locating_pipe_build_refusals( + *, pipe_code: str, domain_code: str, source: str | None, elaboration: ElaborationMetadata | None +) -> Generator[None, None, None]: """Let a refusal raised while building one pipe leave the load loop located on that pipe and its file. The loop is the one place that holds the pipe's code, its domain and the file it is declared in at @@ -119,19 +149,31 @@ def _locating_pipe_build_refusals(*, pipe_code: str, domain_code: str, source: s no-verdict fault, a security refusal is never absorbed into a verdict, a library error keeps the structured items its own arm forwards, and pydantic's ``ValidationError`` and the ``PipeValidationError`` family (not ``PipelexError``s) keep their categorizers. + + A synthetic helper the bundle elaborator generated (the ``__draft_text`` and ``__structure`` + pipes of a ``preliminary_text`` PipeLLM) is not in the author's file, so its refusal is located on + the authored pipe instead, and an unknown model on the authored field the helper's model came from. """ try: yield except PipeOperatorModelChoiceError as model_choice_error: if model_choice_error.source is None: model_choice_error.source = source - raise + if elaboration is None: + raise + relocated = _relocate_on_authored_pipe(model_choice_error=model_choice_error, elaboration=elaboration, domain_code=domain_code) + if relocated is model_choice_error: + raise + raise relocated from model_choice_error except (SecurityError, LibraryError): raise except PipelexError as refusal: if not error_domain_is_input(refusal.to_error_report().error_domain): raise - raise PipeLoadRefusalError.make_from_refusal(refusal=refusal, pipe_code=pipe_code, domain_code=domain_code, source=source) from refusal + authored_pipe_code = elaboration.parent_pipe_code if elaboration is not None else pipe_code + raise PipeLoadRefusalError.make_from_refusal( + refusal=refusal, pipe_code=authored_pipe_code, domain_code=domain_code, source=source + ) from refusal class LibraryManager(LibraryManagerAbstract): @@ -564,7 +606,9 @@ def load_from_crate(self, *, library_id: str, crate: LibraryCrate) -> list[PipeA concept_codes_for_domain = domain_concept_codes.get(domain_code, []) source = crate.source_map.get(pipe_ref) - with _locating_pipe_build_refusals(pipe_code=pipe_code, domain_code=domain_code, source=source): + with _locating_pipe_build_refusals( + pipe_code=pipe_code, domain_code=domain_code, source=source, elaboration=crate.elaboration_metadata.get(pipe_ref) + ): pipe = PipeFactory[PipeAbstract].make_from_blueprint( domain_code=domain_code, pipe_code=pipe_code, @@ -1169,12 +1213,20 @@ def _load_single_dependency( if has_exports and pipe_code not in all_exported: continue try: - pipe = PipeFactory[PipeAbstract].make_from_blueprint( - domain_code=domain_code, + # The same location the main load path attaches, so a dependency pipe's refusal + # names the dependency's own file. + with _locating_pipe_build_refusals( pipe_code=pipe_code, - blueprint=pipe_blueprint, - concept_codes_from_the_same_domain=domain_concept_codes.get(domain_code, []), - ) + domain_code=domain_code, + source=crate.source_map.get(pipe_ref), + elaboration=crate.elaboration_metadata.get(pipe_ref), + ): + pipe = PipeFactory[PipeAbstract].make_from_blueprint( + domain_code=domain_code, + pipe_code=pipe_code, + blueprint=pipe_blueprint, + concept_codes_from_the_same_domain=domain_concept_codes.get(domain_code, []), + ) child_library.pipe_library.add_new_pipe(pipe=pipe) except ValidationError as exc: log.warning(f"Could not load dependency '{alias}' pipe '{pipe_code}': {exc}") diff --git a/pipelex/pipeline/fixes/fix_render.py b/pipelex/pipeline/fixes/fix_render.py index 7fa4dcfd5a..8e71267e63 100644 --- a/pipelex/pipeline/fixes/fix_render.py +++ b/pipelex/pipeline/fixes/fix_render.py @@ -106,10 +106,10 @@ def format_fix_still_invalid_markdown(result: FixBundleResult, *, bundle_path: s if result.remaining_errors: lines += ["", format_validation_error_items_markdown(result.remaining_errors)] - # A remaining error can still carry a 💡 suggested-fix line — dropped by --select/--ignore, left + # A remaining error can still carry a safe 💡 suggested-fix line — dropped by --select/--ignore, left # outside the write scope, or unconverged when the loop bailed. Claiming "no safe fix" there would # contradict the line just printed, so only say it when nothing fixable remains (mirrors the human tip). - if any(item.suggested_fix is not None for item in result.remaining_errors): + if any(item.suggested_fix is not None and item.suggested_fix.safety.is_safe for item in result.remaining_errors): lines += [ "", ( diff --git a/pipelex/pipeline/fixes/planner.py b/pipelex/pipeline/fixes/planner.py index 71a1fb787e..7a4e02fea5 100644 --- a/pipelex/pipeline/fixes/planner.py +++ b/pipelex/pipeline/fixes/planner.py @@ -17,13 +17,15 @@ # Every fix-rule code the planner can emit — the validation set for user-facing rule filters # (``--select`` / ``--ignore``). A new rule constant above must be added here; the CLI rejects # codes outside this set loudly (a typo'd filter selects *behavior*, so lenient-ignore is wrong). +# The codes `pipelex fix bundle --select/--ignore` accept: the rules the fix loop can apply, all SAFE. +# `rename-model` is left out on purpose: it is UNSAFE, so the loop never applies it, and accepting it in +# `--select` would promise a fix the command then silently skips. KNOWN_FIX_CODES: frozenset[str] = frozenset( { MATCH_SEQUENCE_OUTPUT_FIX_CODE, SYNC_CONTROLLER_INPUTS_FIX_CODE, STRIP_NATIVE_CONCEPT_REDECL_FIX_CODE, STRIP_NAMESPACE_FIX_CODE, - RENAME_MODEL_FIX_CODE, } ) diff --git a/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py index 8c3cf997d5..334c105c79 100644 --- a/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py +++ b/tests/integration/pipelex/pipeline/test_validate_bundle_load_refusals.py @@ -175,6 +175,30 @@ class _UnknownModelCase(NamedTuple): ] +def _preliminary_text_bundle(*, model_line: str) -> str: + """A PipeLLM the elaborator splits into ``extract_tide__draft_text`` and ``extract_tide__structure`` helpers.""" + return f""" +domain = "{_DOMAIN}" +description = "Read the tide fact the harbour board publishes" +main_pipe = "extract_tide" + +[concept.TideFact] +description = "When the tide turns at one harbour" + +[concept.TideFact.structure] +harbour = {{ type = "text", description = "The harbour's name", required = true }} + +[pipe.extract_tide] +type = "PipeLLM" +description = "Extract the tide fact from the harbour board's notice" +inputs = {{ tide_times = "Text" }} +output = "TideFact" +structuring_method = "preliminary_text" +{model_line} +prompt = "Extract the tide fact from this notice: $tide_times" +""" + + def _write_bundle(*, directory: Path, content: str) -> Path: bundle_path = directory / "bundle.mthds" bundle_path.write_text(content, encoding="utf-8") @@ -246,6 +270,30 @@ async def test_one_suggestion_carries_a_rename_fix(self, tmp_path: Path) -> None assert fix.source == str(bundle_path) assert fix.ops == [RemapValueOp(table_path=["pipe", "write_tide_note"], key="model", mapping={"@best-sonet": "@best-gpt"})] + @pytest.mark.parametrize( + ("model_line", "field_name"), + [ + ('model = "@best-sonet"', "model"), + ('model_to_structure = "@best-sonet"', "model_to_structure"), + ], + ids=["draft_text_helper", "structure_helper"], + ) + async def test_preliminary_text_helper_refusal_is_located_on_the_authored_pipe(self, tmp_path: Path, model_line: str, field_name: str) -> None: + """The elaborator's helper pipes are not in the author's file, so the item and its fix name the authored pipe and field.""" + bundle_path = _write_bundle(directory=tmp_path, content=_preliminary_text_bundle(model_line=model_line)) + + item = await _single_item(bundle_path=bundle_path) + + assert item.error_type == PipeValidationErrorType.UNKNOWN_MODEL + assert item.pipe_code == "extract_tide" + assert item.field_name == field_name + assert item.field_path == f"pipe.extract_tide.{field_name}" + assert item.source == str(bundle_path) + assert item.message.startswith(f"Pipe 'extract_tide' (PipeLLM), field '{field_name}': ") + fix = item.suggested_fix + assert fix is not None + assert fix.ops == [RemapValueOp(table_path=["pipe", "extract_tide"], key=field_name, mapping={"@best-sonet": "@best-gpt"})] + async def test_several_suggestions_carry_no_fix(self, tmp_path: Path) -> None: bundle_path = _write_bundle(directory=tmp_path, content=_llm_bundle(model_line='model = "$writting-factual"')) diff --git a/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py b/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py index 0994db8ca3..d925ec66ab 100644 --- a/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py +++ b/tests/unit/pipelex/pipeline/fixes/test_fix_planner.py @@ -11,7 +11,7 @@ import pytest from pipelex.core.exceptions import PipesAndConceptValidationErrorData -from pipelex.pipeline.fixes.planner import RENAME_MODEL_FIX_CODE, plan_fix_for_pipe_validation_error +from pipelex.pipeline.fixes.planner import KNOWN_FIX_CODES, RENAME_MODEL_FIX_CODE, plan_fix_for_pipe_validation_error from pipelex.suggested_fix import DeleteKeyOp, EnsureTableOp, FixSafety, RemapValueOp, SetKeyOp from pipelex.validation_error_types import PipeValidationErrorType @@ -216,6 +216,7 @@ def test_unknown_model_with_one_suggestion_yields_rename_model_fix(self) -> None assert fix is not None assert fix.fix_code == RENAME_MODEL_FIX_CODE assert fix.safety == FixSafety.UNSAFE + assert RENAME_MODEL_FIX_CODE not in KNOWN_FIX_CODES, "an unsafe code is never selectable: the fix loop would skip it" assert fix.source == "main.mthds" assert fix.ops == [RemapValueOp(table_path=["pipe", "write_tide_note"], key="model", mapping={"@best-sonet": "@best-gpt"})] assert "'@best-sonet'" in fix.description From 18dd03d8b9c6fbd55751ea09677ba6ac31d48844 Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 26 Sep 2026 19:36:04 +0200 Subject: [PATCH 4/5] Pin the locating of refusals raised while building dependency pipes A dependency package's pipes are built outside the main load loop. These tests load one through `_load_single_dependency` and check that an unknown model names the dependency's file, that a preliminary_text helper's refusal lands on the authored pipe and field, and that any other input refusal becomes a PipeLoadRefusalError located on the pipe. Each test fails when the locating wrapper is removed. Co-Authored-By: Claude Opus 5.5 --- .../test_dependency_pipe_build_refusals.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 tests/unit/pipelex/libraries/test_dependency_pipe_build_refusals.py diff --git a/tests/unit/pipelex/libraries/test_dependency_pipe_build_refusals.py b/tests/unit/pipelex/libraries/test_dependency_pipe_build_refusals.py new file mode 100644 index 0000000000..4d1454f325 --- /dev/null +++ b/tests/unit/pipelex/libraries/test_dependency_pipe_build_refusals.py @@ -0,0 +1,112 @@ +from pathlib import Path + +import pytest +from mthds.package.dependency_resolver import ResolvedDependency +from mthds.package.manifest.schema import MethodsManifest +from pytest_mock import MockerFixture + +from pipelex.base_exceptions import ErrorDomain, PipelexError +from pipelex.core.pipes.exceptions import PipeLoadRefusalError, PipeOperatorModelChoiceError +from pipelex.interpreter_hub import get_library_manager +from pipelex.libraries.library_factory import LibraryFactory +from pipelex.libraries.library_manager import LibraryManager + +_DEP_DOMAIN = "harbour_dep" + +_SCORE_PIPE = """ +[pipe.score_tide] +type = "PipeLLM" +description = "Score how favourable the tide is for leaving harbour" +inputs = {{ tide_times = "Text" }} +output = "Text" +{model_line} +prompt = "Score how favourable these tide times are for leaving harbour: $tide_times" +""" + +_PRELIMINARY_TEXT_PIPE = """ +[concept.TideFact] +description = "When the tide turns at one harbour" + +[concept.TideFact.structure] +harbour = { type = "text", description = "The harbour's name", required = true } + +[pipe.extract_tide] +type = "PipeLLM" +description = "Extract the tide fact from the harbour board's notice" +inputs = { tide_times = "Text" } +output = "TideFact" +structuring_method = "preliminary_text" +model_to_structure = "@best-sonet" +prompt = "Extract the tide fact from this notice: $tide_times" +""" + + +def _dep_bundle(*, pipes: str) -> str: + return f'domain = "{_DEP_DOMAIN}"\ndescription = "A dependency package the harbour board publishes"\n{pipes}' + + +class _InternalInputRefusalError(PipelexError): + error_domain = ErrorDomain.INPUT + + +class TestDependencyPipeBuildRefusals: + def _load_dependency(self, *, mocker: MockerFixture, tmp_path: Path, bundle: str) -> Path: + """Load one dependency package through ``_load_single_dependency``, returning its bundle file.""" + mthds_file = tmp_path / "harbour_dep.mthds" + mthds_file.write_text(bundle, encoding="utf-8") + resolved_dep = ResolvedDependency( + alias="harbour_dep", + address="github.com/harbour-board/harbour-dep", + manifest=MethodsManifest(address="github.com/harbour-board/harbour-dep", version="1.0.0", description="A dependency package"), + package_root=tmp_path, + mthds_files=[mthds_file], + exported_pipe_codes=None, + ) + library = LibraryFactory.make_empty() + mocker.patch.object(get_library_manager(), "get_current_library", return_value=library) + LibraryManager()._load_single_dependency( # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage] + library=library, + resolved_dep=resolved_dep, + ) + return mthds_file + + def test_unknown_model_in_a_dependency_pipe_names_the_dependency_file(self, mocker: MockerFixture, tmp_path: Path) -> None: + """A dependency pipe is built outside the main load loop, and its refusal still carries the file it is declared in.""" + bundle = _dep_bundle(pipes=_SCORE_PIPE.format(model_line='model = "@best-sonet"')) + + with pytest.raises(PipeOperatorModelChoiceError) as raised: + self._load_dependency(mocker=mocker, tmp_path=tmp_path, bundle=bundle) + + refusal = raised.value + assert refusal.pipe_code == "score_tide" + assert refusal.domain_code == _DEP_DOMAIN + assert refusal.field_name == "model" + assert refusal.model_choice == "@best-sonet" + assert refusal.source == str(tmp_path / "harbour_dep.mthds") + + def test_unknown_model_in_a_dependency_helper_names_the_authored_pipe(self, mocker: MockerFixture, tmp_path: Path) -> None: + """A ``preliminary_text`` helper of a dependency pipe is reported on the pipe and field its author wrote.""" + with pytest.raises(PipeOperatorModelChoiceError) as raised: + self._load_dependency(mocker=mocker, tmp_path=tmp_path, bundle=_dep_bundle(pipes=_PRELIMINARY_TEXT_PIPE)) + + refusal = raised.value + assert refusal.pipe_code == "extract_tide" + assert refusal.field_name == "model_to_structure" + assert refusal.source == str(tmp_path / "harbour_dep.mthds") + + def test_other_input_refusal_in_a_dependency_pipe_is_located(self, mocker: MockerFixture, tmp_path: Path) -> None: + """Any other refusal of the caller's input raised while building a dependency pipe is located on that pipe and file.""" + mocker.patch( + "pipelex.pipe_operators.llm.pipe_llm.check_llm_choice_with_deck", + side_effect=_InternalInputRefusalError("internal detail: registry slot 7 is stale"), + ) + bundle = _dep_bundle(pipes=_SCORE_PIPE.format(model_line='model = "@best-gpt"')) + + with pytest.raises(PipeLoadRefusalError) as raised: + self._load_dependency(mocker=mocker, tmp_path=tmp_path, bundle=bundle) + + refusal = raised.value + assert refusal.pipe_code == "score_tide" + assert refusal.domain_code == _DEP_DOMAIN + assert refusal.source == str(tmp_path / "harbour_dep.mthds") + assert isinstance(refusal.__cause__, _InternalInputRefusalError) From 5312fccad256653e8dd8ef81982ea24f7cc1fa05 Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 26 Sep 2026 20:05:11 +0200 Subject: [PATCH 5/5] Review round 3: one account of the selectable fix codes The comment above KNOWN_FIX_CODES still said it held every code the planner can emit while the lines below it said rename-model is left out on purpose. It now says what the set is: the SAFE rules the fix loop can apply, with an UNSAFE rule such as rename-model kept out. Co-Authored-By: Claude Opus 5.5 --- pipelex/pipeline/fixes/planner.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pipelex/pipeline/fixes/planner.py b/pipelex/pipeline/fixes/planner.py index 7a4e02fea5..1f179667ef 100644 --- a/pipelex/pipeline/fixes/planner.py +++ b/pipelex/pipeline/fixes/planner.py @@ -14,12 +14,11 @@ STRIP_NAMESPACE_FIX_CODE = "strip-namespace" RENAME_MODEL_FIX_CODE = "rename-model" -# Every fix-rule code the planner can emit — the validation set for user-facing rule filters -# (``--select`` / ``--ignore``). A new rule constant above must be added here; the CLI rejects -# codes outside this set loudly (a typo'd filter selects *behavior*, so lenient-ignore is wrong). -# The codes `pipelex fix bundle --select/--ignore` accept: the rules the fix loop can apply, all SAFE. -# `rename-model` is left out on purpose: it is UNSAFE, so the loop never applies it, and accepting it in -# `--select` would promise a fix the command then silently skips. +# The codes `pipelex fix bundle --select/--ignore` accept: every SAFE rule the fix loop can apply. A new +# SAFE rule constant above must be added here; the CLI rejects codes outside this set loudly (a typo'd +# filter selects *behavior*, so lenient-ignore is wrong). An UNSAFE rule stays out: the loop never applies +# it, so accepting it in `--select` would promise a fix the command then silently skips. `rename-model` +# is the one such rule today. KNOWN_FIX_CODES: frozenset[str] = frozenset( { MATCH_SEQUENCE_OUTPUT_FIX_CODE,