refactor(cli): bad input exits 2 through one InputError

dokimi/changelog.d/20261030_120000_napetrov_arch-loader.md

@@ -0,0 +1,23 @@
+### Changed
+
+- **Bad input exits 2 in the `novelty` and `report model-compare` commands** — a
+  file that `novelty` cannot read or that is not the artifact it names (`--artifact`,
+  `--atoms`, `--probes`, `--competitors`, `--queries-by-skill`), and the runs that
+  `report model-compare` refuses (a missing, unreadable, non-JSON or non-object file;
+  runs that disagree on the treatment, prompt or judge), and the skill fields that
+  `novelty routing` refuses (a missing competitor, a skill listed twice, a skill compared
+  with itself, an all-vs-all catalog that holds one skill twice with different content),
+  exit 2 with `Error: <message>` on stderr. For a bad file the message names the file and
+  the reason; for a refused field it names the skill or option at fault. The `novelty`
+  file refusals keep the exit code 2 they had, `--artifact` and `--atoms` among them; the
+  `model-compare`, `routing` and `view` refusals keep that code too. Exit 1 stays the verdict of the gates
+  (`--strict`, `eval judge-validity`) and the code of a model failure (an extractor or judge
+  that errors), so a script that branches on the code must treat 2 as "fix the input".
+  `subjects run --novelty-artifact` still prints its own `Error loading novelty artifact:` message
+  (now with the new reason text) and exits 2, as before. `novelty view`
+  also checks the `bundle` of an atoms file, so a malformed bundle is refused with exit 2. Two `report model-compare` messages changed wording:
+  `File not found: <file>` is now `cannot read <file>: No such file or directory`, and
+  `Invalid JSON in <file>` is now `<file> is not valid JSON`. An invalid artifact named by
+  `novelty view` says which part is wrong: `is not valid JSON`, `is not UTF-8 text`,
+  `is not a novelty artifact: expected a JSON object`, or `is not a valid novelty artifact:
+  <reason>`.

dokimi/docs/architecture.md

@@ -420,6 +420,10 @@ Guide: [run-consistency.md](run-consistency.md).
   decides who owns each pipe, so an interrupted launch closes its streams exactly once
   and Windows reader threads close theirs at EOF. `FakeRunner` is its test adapter: code that spawns
   takes a `runner` argument.
+- Bad input: a library loader raises `dokimi.utils.InputError`, naming the file, and
+  never ends the process. `cli.main` prints `Error: <message>` and exits 2; exit 1 stays
+  the verdict of a gate. `tests/test_import_architecture.py` allows an exit only in the
+  frontend.
 - `dokimi/llm.py` — the entry point for LLM calls. `LLM` (one model at one provider:
   `complete`/`ask`/`chat`, retry, the response cache, usage) sits on one
   adapter seam: `LiteLLMAdapter` in production, `FakeAdapter` in tests (the

dokimi/dokimi/artifacts.py

@@ -34,6 +34,8 @@
 from pathlib import Path
 from typing import Any
 
+from dokimi.utils import InputError, read_json_input
+
 logger = logging.getLogger(__name__)
 
 SCHEMA_DIR = Path(__file__).resolve().parent / "schemas"
@@ -207,11 +209,18 @@ def _write_target(path: Path) -> Path:
     return target
 
 
-def load_artifact(kind: str, path: Path, *, validate: bool = True) -> dict:
-    """Read and (optionally) validate an artifact from ``path``."""
-    data = json.loads(Path(path).read_text(encoding="utf-8"))
+def load_artifact(kind: str, path: Path | str, *, validate: bool = True) -> dict:
+    """Read and (optionally) validate an artifact of ``kind`` from ``path``.
+
+    Every failure is an :class:`~dokimi.utils.InputError` naming ``path``: a file that is
+    missing, unreadable, not UTF-8 or JSON, not a JSON object, or that fails the schema.
+    """
+    data = read_json_input(path)
     if not isinstance(data, dict):
-        raise ValueError(f"Artifact must be a JSON object, got {type(data).__name__}")
+        raise InputError(f"{path} is not a {kind} artifact: expected a JSON object, got {type(data).__name__}")
     if validate:
-        validate_artifact(kind, data)
+        try:
+            validate_artifact(kind, data)
+        except ArtifactValidationError as exc:
+            raise InputError(f"{path} is not a valid {kind} artifact: {exc}") from exc
     return data

dokimi/dokimi/cli.py

@@ -193,15 +193,14 @@ def main() -> None:
         process.install_signal_handlers()  # SIGTERM/SIGHUP: kill the process groups we started, then exit
     tolerate_console_encoding()
     args = parse_args()
-    from dokimi.eval.question_text import EmptyQuestionError
     from dokimi.utils import InputError
 
     try:
         args.func(args)
-    except (EmptyQuestionError, InputError, llm_cli.SamplingError) as exc:  # bad input, not a bug: no traceback
+    except (InputError, llm_cli.SamplingError) as exc:  # bad input, not a bug: no traceback
         print(f"Error: {exc}", file=sys.stderr)
-        # 2, as the commands' other input errors: 1 is the verdict of `eval judge-validity`
-        # and of the --strict gates, and a malformed file must not read as a failed gate.
+        # 2 is the one code for bad input: 1 is the verdict of `eval judge-validity` and of the
+        # --strict gates, and a malformed file must not read as a failed gate.
         sys.exit(2)
 
 

dokimi/dokimi/commands/_output.py

@@ -20,3 +20,16 @@ def exit_on_unwritable(path: Path):
     except OSError as exc:
         print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
         sys.exit(EXIT_USAGE)
+
+
+def write_text_or_exit(path: Path, text: str, code: int = EXIT_USAGE) -> None:
+    """Write ``text`` to ``path`` (creating parents), or exit ``code`` naming ``path`` when it cannot be written.
+
+    A command helper: library code that cannot write raises, and the command decides the exit.
+    """
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+    except OSError as exc:
+        print(f"Error: cannot write {path}: {exc}", file=sys.stderr)
+        sys.exit(code)

dokimi/dokimi/commands/arms.py

@@ -14,7 +14,7 @@
 
 from dokimi.exit_codes import EXIT_FAILURE, EXIT_USAGE
 from dokimi.llm import ANSWER_PROVIDERS
-from dokimi.utils import question_id
+from dokimi.utils import InputError, question_id
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.defaults import (
     DEFAULT_ANSWER_MODEL,
@@ -123,8 +123,8 @@ def _load_matrix_inputs(config: ArmsRunConfig, arms: str, questions: str, matrix
         cells = load_matrix_cells(matrix_cells)
         _validate_unique_cell_filenames(cells)
         arms_run.check_matrix(config, cells, specs)
-    except EmptyQuestionError:
-        raise  # a bad input record, not a cell error: cli.main reports it (exit 2)
+    except InputError:  # EmptyQuestionError is one
+        raise  # a bad input record or a refused cell: cli.main reports it (exit 2)
     except arms_run.ProbeError as exc:
         print(f"Error loading matrix run inputs: {exc}", file=sys.stderr)
         sys.exit(EXIT_FAILURE)

dokimi/dokimi/commands/evaluate.py

@@ -23,8 +23,9 @@
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs
-from dokimi.utils import same_evaluator, write_text_or_exit
+from dokimi.commands._output import write_text_or_exit
 from dokimi.exit_codes import EXIT_USAGE
+from dokimi.utils import same_evaluator
 
 
 def _warn_judge_independence(
@@ -47,14 +48,10 @@ def _warn_judge_independence(
 
 
 def _load_answers(path: str):
-    """The answers artifact at ``path`` (read by dokimi/records.py), or a clean exit."""
-    from dokimi.records import RecordError, load
+    """The answers artifact at ``path`` (read by dokimi/records.py); a RecordError (an InputError) if not."""
+    from dokimi.records import load
 
-    try:
-        return load(path, section="answers")
-    except RecordError as exc:
-        print(f"Error: {exc}", file=sys.stderr)
-        sys.exit(2)
+    return load(path, section="answers")
 
 
 def _write_json_or_exit(path: Path, payload: dict, code: int = EXIT_USAGE) -> None:

dokimi/dokimi/commands/explore.py

@@ -133,7 +133,7 @@ def cmd_explore_run(args: argparse.Namespace) -> None:
 
     if args.out_md:
         from dokimi.report.exploration_report import render_explore_report
-        from dokimi.utils import write_text_or_exit
+        from dokimi.commands._output import write_text_or_exit
 
         write_text_or_exit(Path(args.out_md), render_explore_report(output))
         print(f"Saved exploration report: {args.out_md}")

dokimi/dokimi/commands/judge_reliability.py

@@ -19,34 +19,28 @@
 )
 from dokimi.eval.judge import JUDGE_PROVIDERS
 from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY
+from dokimi.utils import InputError, read_json_input
+from dokimi.commands._output import write_text_or_exit
 from dokimi.exit_codes import EXIT_FAILURE, EXIT_USAGE
-from dokimi.utils import write_text_or_exit
 
 if TYPE_CHECKING:
     from dokimi.eval.cross_judge import Measure
 
 
 def _read_json(path: str):
-    """The parsed JSON at ``path``, or exit 2 (bad input, not a failed gate) naming the file."""
-    try:
-        return json.loads(Path(path).read_text(encoding="utf-8"))
-    except OSError as exc:
-        print(f"Error: cannot read {path}: {exc}", file=sys.stderr)
-    except (ValueError, RecursionError) as exc:  # JSONDecodeError, bad UTF-8, deep nesting
-        print(f"Error: {path} is not valid JSON: {exc}", file=sys.stderr)
-    sys.exit(2)
+    """The parsed JSON at ``path``; an InputError naming the file when it cannot be read (exit 2)."""
+    return read_json_input(path)
 
 
 def _load_json_object(path: str, key: str, kind: type = dict) -> dict:
-    """The JSON object at ``path``, or exit 2 naming the file.
+    """The JSON object at ``path``, or InputError naming the file.
 
     The object must also hold a ``kind`` (object or list) under ``key``.
     """
     data = _read_json(path)
     if not isinstance(data, dict) or not isinstance(data.get(key), kind):
         what = f"a JSON object with a `{key}` {'list' if kind is list else 'object'}"
-        print(f"Error: {path} is not {what}", file=sys.stderr)
-        sys.exit(2)
+        raise InputError(f"{path} is not {what}")
     return data
 
 
@@ -57,40 +51,33 @@ def _judge_name(doc: dict, path: str) -> str:
 
 
 def _result_measure(path: str, key: str, kind: type, build):
-    """The measure an `eval pairwise` or `eval checklist` file gives, or exit 2 naming the file.
+    """The measure an `eval pairwise` or `eval checklist` file gives, or an InputError naming the file.
 
     *build* is ``(doc, name) -> Measure``; it raises ValueError on an entry of the wrong shape.
     """
     doc = _load_json_object(path, key, kind)
     try:
         return build(doc, _judge_name(doc, path))
     except ValueError as exc:
-        print(f"Error: {path}: {exc}", file=sys.stderr)
-        sys.exit(2)
+        raise InputError(f"{path}: {exc}") from exc
 
 
 def _load_answers(path: str):
-    """The parsed file and its answers records (dokimi/records.py), or a clean exit."""
-    from dokimi.records import RecordError, read
+    """The parsed file and its answers records (dokimi/records.py); a RecordError (an InputError) if not."""
+    from dokimi.records import read
 
     data = _read_json(path)
-    try:
-        answers = list(read(data, source=path, section="answers").records)
-    except RecordError as exc:
-        print(f"Error: {exc}", file=sys.stderr)
-        sys.exit(2)  # 1 is the verdict code of judge-validity and the --strict gates
-    return data, answers
+    return data, list(read(data, source=path, section="answers").records)
 
 
 def _load_points(path: str):
-    """question_id -> required points from a questions file, or exit 2 naming the file."""
+    """question_id -> required points from a questions file, or an InputError naming the file."""
     from dokimi.eval.checklist import load_points
 
     try:
         return load_points(path)
     except (OSError, ValueError) as exc:  # ValueError covers bad JSON, bad UTF-8 and a refused set
-        print(f"Error: cannot read questions {path}: {exc}", file=sys.stderr)
-        sys.exit(2)
+        raise InputError(f"cannot read questions {path}: {exc}") from exc
 
 
 def _require_arms(path: str, records, command: str) -> None:
@@ -102,10 +89,9 @@ def _require_arms(path: str, records, command: str) -> None:
     from dokimi.records import has_arms
 
     if not has_arms(records):
-        print(f"Error: {path} has no arms answers (no record has an `arms` field); "
-              f"eval {command} reads an arms run (arms run --out-json), not a binary "
-              "with_docs/without_docs answers file", file=sys.stderr)
-        sys.exit(2)
+        raise InputError(f"{path} has no arms answers (no record has an `arms` field); "
+                         f"eval {command} reads an arms run (arms run --out-json), not a binary "
+                         "with_docs/without_docs answers file")
 
 
 def _progress(label: str = ""):