diff --git a/.agent-eval/Dockerfile b/.agent-eval/Dockerfile index 445e8e1b2..9fc3a7b6e 100644 --- a/.agent-eval/Dockerfile +++ b/.agent-eval/Dockerfile @@ -2,9 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # Build image for dokimi's own repository-mode evals: CI's Python, the lock, every CI tool, -# and a warm uv cache so `make dev` inside a run works without downloading anything. -# Pinned by digest, like the tools below, so a rerun grades in the same image. To move it: -# `docker buildx imagetools inspect python:3.11-slim-bookworm`, then `repo tasks check`. +# pinned by digest. To move the pin: `docker buildx imagetools inspect python:3.11-slim-bookworm`, then `eval check`. FROM python:3.11-slim-bookworm@sha256:0a310eeecf4e1f5a0743f9a6520c90c88d089c903ca5fd283f501e3a805f5f89 # Debian's point releases replace these package versions, so pinning them would break the build. # hadolint ignore=DL3008 @@ -17,6 +15,7 @@ COPY dokimi/requirements.lock.txt dokimi/requirements-ci.lock.txt dokimi/require # The lock and the tools CI installs (uv among them), hash-checked so a rerun grades alike. RUN pip install --no-cache-dir --require-hashes -r /opt/dokimi-req/requirements.lock.txt \ -r /opt/dokimi-req/requirements-ci.lock.txt +# A warm uv cache, so `make dev` inside a run downloads nothing. ENV UV_CACHE_DIR=/opt/uv-cache UV_PYTHON_PREFERENCE=only-system UV_LINK_MODE=copy RUN uv venv --python 3.11 --seed /tmp/warm && uv pip install --python /tmp/warm/bin/python \ --require-hashes -r /opt/dokimi-req/requirements.lock.txt -r /opt/dokimi-req/requirements-ci.lock.txt \ diff --git a/.agent-eval/README.md b/.agent-eval/README.md index 7c980283f..a56b062e3 100644 --- a/.agent-eval/README.md +++ b/.agent-eval/README.md @@ -42,11 +42,11 @@ Process metrics are reported and never gated. They come from `events.jsonl` and ## Running -`repo tasks check` and `repo run` need Docker. Run them from `dokimi/`: +`eval check` and `repo run` need Docker. Run them from `dokimi/`: ```bash # self-check, no LLM: the untouched workspace fails each gate, the reference solution passes it -python cli.py repo tasks check .. --manifest .agent-eval/repo-eval.yaml -j 4 +python cli.py eval check .. --manifest .agent-eval/repo-eval.yaml -j 4 # agents: one cell per (task, arm, model, rep) python cli.py repo run .. --manifest .agent-eval/repo-eval.yaml \ @@ -79,4 +79,4 @@ A task is a directory under `tasks/` with these files: Write the prompt the way a maintainer would ask, and leave out the repository's conventions: those are what the guidance is measured on. -A task is ready when `repo tasks check` reports it `OK`. +A task is ready when `eval check` reports it `OK`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc4abe517..a6bab6325 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -399,9 +399,8 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.13' - # The `[ragas]` extra is not in requirements.txt or the lock; this job is - # where `eval ragas` runs on real packages, and DOKIMI_REQUIRE_RAGAS=1 makes - # tests/test_ragas_runtime.py fail rather than skip if ragas is missing. + # The `[ragas]` extra is not in requirements.txt or the lock; this job is where + # `eval score ragas` runs on real packages (DOKIMI_REQUIRE_RAGAS=1 fails, not skips). - name: Install newest allowed deps run: pip install --upgrade -r requirements.txt -r requirements-test.txt -e '.[ragas]' - name: Run tests diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index a44766dc0..e1f56dd95 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -3,26 +3,8 @@ name: repo-eval -# The no-LLM checks of a repository's evals (dokimi/docs/repo-evaluation-contract.md), -# for repositories to call from their own workflow: -# -# validate `repo validate`: the manifest and every task.json (seconds; a gate) -# scan `repo scan`: the L0 claims of the guidance (report only) -# tasks-check `repo tasks check`, one job per selected task: the untouched -# workspace fails the gate, the reference solution passes it -# -# Nothing here calls a model or reads a secret. The caller pins dokimi twice, -# by the same commit SHA (or tag): in `uses: ...@` and in `dokimi-ref`. The -# ref installs dokimi and, through its requirements.lock.txt, every dependency, -# so a rerun checks with the same framework. A branch name is refused. The -# calling job grants `contents: read`, all any job here asks for. -# -# Task selection (`tasks`): `changed` on a pull request is the tasks whose -# directories it touches, plus `smoke-tasks` when it changes anything else the -# self-check depends on: the eval directory outside its tasks (harness, -# graders, Dockerfile; Markdown excepted), the manifest, the calling workflow -# file, or a path under `smoke-paths`. On a push `changed` diffs against the -# previous tip; on any other event it means `all`. +# The no-LLM checks of a repo's evals, as a reusable workflow: no model, no secret. +# Jobs, task selection and the pin rule: dokimi/docs/repo-evaluation-contract.md, "CI for a repository". on: workflow_call: @@ -74,14 +56,14 @@ on: default: 4 warm: description: >- - Pass --warm to `repo tasks check`: null and oracle run with network so a + Pass --warm to `eval check`: null and oracle run with network so a cold cache fills (an ephemeral runner starts with empty caches) required: false type: boolean default: false allow-unsupported: description: >- - Pass --allow-unsupported to `repo tasks check`: a task this runner misses the task.json + Pass --allow-unsupported to `eval check`: a task this runner misses the task.json `host` requirements of passes its job (the reason is still in the log and summary) required: false type: boolean @@ -375,7 +357,7 @@ jobs: pip install --no-deps "dokimi @ git+https://github.com/$DOKIMI_REPO@$SHA#subdirectory=dokimi" # Builds the image from the manifest's Dockerfile (the tree without its # tasks), then grades the null and the oracle workspace in it. - - name: repo tasks check + - name: eval check env: MANIFEST: ${{ inputs.manifest }} TASK: ${{ matrix.task }} @@ -387,7 +369,7 @@ jobs: flags=() if [ "$WARM" = true ]; then flags+=(--warm); fi if [ "$ALLOW_UNSUPPORTED" = true ]; then flags+=(--allow-unsupported); fi - dokimi repo tasks check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${flags[@]}" + dokimi eval check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${flags[@]}" # always(): a hung oracle ends in the job timeout, which cancels the job; a # cancelled job still runs these steps, so the run leaves a verdict. - name: Summary diff --git a/dokimi/CONTRIBUTING.md b/dokimi/CONTRIBUTING.md index f95aa84a0..6393002c7 100644 --- a/dokimi/CONTRIBUTING.md +++ b/dokimi/CONTRIBUTING.md @@ -49,7 +49,7 @@ The package exposes a console entry point: dokimi --help # equivalent to `python -m dokimi` ``` -Optional feature sets are available as extras: `ragas` (`eval ragas`, which +Optional feature sets are available as extras: `ragas` (`eval score ragas`, which `dev` includes), `ocr` (Docling ingestion), `mcp`, and `embeddings`. Install what you need, e.g. `pip install -e ".[dev,ocr]"`. diff --git a/dokimi/README.md b/dokimi/README.md index fd14399c1..b0c6146f2 100644 --- a/dokimi/README.md +++ b/dokimi/README.md @@ -67,7 +67,7 @@ The project is pre-1.0; interfaces and artifact schemas may still change. `AWS_REGION` + `AWS_BEARER_TOKEN_BEDROCK` for Bedrock). `GITHUB_TOKEN` is optional and raises the persona-discovery rate limit. - Docker, only for the executable-task, harness and repository tracks. -- Optional extras: `ragas` (`eval ragas`, `answers generate --ragas`), `ocr` +- Optional extras: `ragas` (`eval score ragas`, `answers generate --ragas`), `ocr` (Docling ingestion) and `mcp` (real MCP servers). ```bash @@ -123,10 +123,10 @@ The top-level CLI is `python cli.py` (or `dokimi` once installed): Context benchmark benchmark Preflight and run the benchmark for registered targets evaluate Run the full pipeline in one command - personas Discover and approve target user personas - questions Generate, analyze, refine, and panel-review questions + eval personas Discover and approve target user personas + eval questions Generate, analyze, refine, and review questions answers Generate context-arm and baseline answers - eval Score answers, run RAGAS, or use a multi-judge panel + eval Build and check eval sets, score answers, calibrate judges arms Compare treatment arms (docs / MCP / skills / profiles) subjects Run a subject descriptor (artifact + suite) into a scorecard @@ -156,7 +156,7 @@ That is the curated set; `python cli.py --help` lists every group. Use `--help` on any command for detailed options, for example: ```bash -python cli.py questions --help +python cli.py eval questions --help python cli.py eval --help python cli.py dashboard generate --help ``` diff --git a/dokimi/SECURITY.md b/dokimi/SECURITY.md index 814a45f60..10a4e7c08 100644 --- a/dokimi/SECURITY.md +++ b/dokimi/SECURITY.md @@ -97,7 +97,7 @@ fix the locks can take: that can create files in the lock directory gets a stale-lock break to meet a reused inode (ext4, xfs). -The locks do not contain ragas: it is the optional `[ragas]` extra (`eval ragas`, +The locks do not contain ragas: it is the optional `[ragas]` extra (`eval score ragas`, `answers generate --ragas`), which installs ragas 0.3.9 and diskcache 5.6.3. Their advisories do not affect Dokimi: diff --git a/dokimi/changelog.d/20261030_160000_napetrov_eval-group-m1.md b/dokimi/changelog.d/20261030_160000_napetrov_eval-group-m1.md new file mode 100644 index 000000000..7d50d2c6f --- /dev/null +++ b/dokimi/changelog.d/20261030_160000_napetrov_eval-group-m1.md @@ -0,0 +1,41 @@ +### Changed + +- **The evaluation commands live under `eval` (#94 M1)** — `personas`, `questions` and the + evaluation leaves of `tasks`, `catalog` and `repo` are now under `dokimi eval`, and the + `eval` leaves are grouped into `eval score`, `eval judge`, `eval questions`, `eval personas` + and `eval triggers`. Every old path still answers, but it prints `moved: dokimi ` + to stderr and exits 2 without running anything, so a script fails loudly and names the new + command. `eval score` stays a live group: bare `eval score` exits 2 with a usage error, and + `eval score --help` lists its leaves. Move the scripts to the new paths before the next release. The arguments are + unchanged. `dokimi repo tasks replay`, `repo run`, `repo report` and `repo compare` stay where + they are. + + | Before | After | + |---|---| + | `tasks inspect` | `eval inspect` | + | `repo tasks check` | `eval check` | + | `tasks list` | `eval tasks list` | + | `personas discover` | `eval personas discover` | + | `personas approve` | `eval personas validate` | + | `questions generate` | `eval questions generate` | + | `questions analyze` | `eval questions analyze` | + | `questions refine` | `eval questions refine` | + | `questions panel-review` | `eval questions review` | + | `questions import` | `eval import` | + | `catalog trigger-set` | `eval triggers generate` | + | `eval score` | `eval score pointwise` | + | `eval pairwise` | `eval score pairwise` | + | `eval checklist` | `eval score checklist` | + | `eval panel-score` | `eval score panel` | + | `eval grounding` | `eval score grounding` | + | `eval ragas` | `eval score ragas` | + | `eval reference` | `eval score reference` | + | `eval judge-validity` | `eval judge validate` | + | `eval judge-wording` | `eval judge wording` | + | `eval cross-judge` | `eval judge agreement` | + | `personas` (bare, or its help) | `eval personas` | + | `questions` (bare, or its help) | `eval questions` | + + The `[free]`/`[paid]` labels in `--help` follow the commands. The `consumers` labels that + `tasks inspect` reports (and the `task_inspect.v1` schema enum) name the new paths. The + `repo tasks check` step of `repo-eval.yml` runs `dokimi eval check`; its job name is unchanged. diff --git a/dokimi/data/README.md b/dokimi/data/README.md index afd960b35..a84e06c09 100644 --- a/dokimi/data/README.md +++ b/dokimi/data/README.md @@ -35,7 +35,7 @@ content repository and are supplied to the framework by path — from `data/`. **Captured runs.** Answer runs, judge scores and static-benchmark snapshots are -*outputs*, so none are committed. `cli.py answers generate`, `cli.py eval score` and +*outputs*, so none are committed. `cli.py answers generate`, `cli.py eval score pointwise` and `cli.py run` write them to the git-ignored scratch dirs (`answers/`, `eval/`, `results/`, `reports/`, `baselines/`); the schema each one must satisfy is locked by `dokimi/schemas/*.json` and by `tests/test_artifacts.py`, not by a committed @@ -54,7 +54,7 @@ Generated artifacts are the other case, and they do not land here. A question-ge run writes to the git-ignored scratch dir, and the result is evaluation content: ```bash -python cli.py questions generate --product oneTBB \ +python cli.py eval questions generate --product oneTBB \ --personas personas/onetbb.json \ --output questions/onetbb.json # git-ignored; commit it to the content repo ``` diff --git a/dokimi/data/skills/dokimi-design-evals/SKILL.md b/dokimi/data/skills/dokimi-design-evals/SKILL.md index 42295f4db..ecb2bb551 100644 --- a/dokimi/data/skills/dokimi-design-evals/SKILL.md +++ b/dokimi/data/skills/dokimi-design-evals/SKILL.md @@ -57,7 +57,7 @@ in [references/example-scenario.md](references/example-scenario.md). Prefer deterministic checks: tests, file and diff assertions, exit codes, parsed output. Use an LLM judge only for qualities no check can capture; then use a written rubric, calibrate it -(`python cli.py eval judge-validity --answers --sequential --strict`), +(`python cli.py eval judge validate --answers --sequential --strict`), and keep a human review of a sample. The command calibrates only dokimi's own answer judge (rows with `question_id`), not a task's rubric judge; check that one by hand. diff --git a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md index 5492eb2ec..759f2d22f 100644 --- a/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md +++ b/dokimi/data/skills/dokimi-design-evals/references/verifier-self-test.md @@ -11,7 +11,7 @@ Repository tasks (`repo-eval.yaml` + `task.json`): python cli.py repo validate --manifest --fail-on-warning ``` -Harbor tasks: there is no validate command. `tasks list` only shows that the +Harbor tasks: there is no validate command. `eval tasks list` only shows that the task was discovered (it reads `task.toml` and `instruction.md`); it never prints an error row, and it lists a task that has no `tests/test.sh`, `solution/` or `environment/`. A task without `instruction.md` aborts the listing with a @@ -25,7 +25,7 @@ out as `verifier/` + `oracle/`, but the Harbor-backed arms (`codex`, the task will run in the paid arms. ```bash -python cli.py tasks list --tasks-root +python cli.py eval tasks list --tasks-root ``` ## 2. Oracle passes, untouched workspace fails, isolation @@ -35,7 +35,7 @@ read tasks or source; `docs/repo-evaluation-contract.md` §4.3). It exits 1 on any broken or unsupported task. ```bash -python cli.py repo tasks check --manifest --task +python cli.py eval check --manifest --task ``` Harbor tasks: oracle and isolation have harnesses; expect `pass_rate: 1.0` @@ -69,7 +69,7 @@ changing only the reference solution so it produces the mutated end state: `pass_rate: 0.0`. - Repository: write a scratch copy of `repo-eval.yaml` whose `oracle` command runs the real oracle and then the mutation, and pass it with - `--manifest-file` to `repo tasks check`. Expect the oracle check to fail. + `--manifest-file` to `eval check`. Expect the oracle check to fail. The oracle command is split into argv and run **without a shell**, so `&&`, pipes and redirects are passed to the program as arguments and the mutation never runs (the check then passes, which looks like a verifier that survived). @@ -94,7 +94,7 @@ a correct answer (a negative task, an out-of-scope request), set `valid_inaction: true`: the null workspace is then expected to **pass**, and the "untouched fails" check does not apply. Use a mutation that does act wrongly instead (an unrequested edit). Do not reword the task so that -inaction fails just to satisfy the template. Note that `repo tasks check` +inaction fails just to satisfy the template. Note that `eval check` always requires the null to fail; a valid-inaction task needs a grader that fails on the unwanted action and its own manual null check. @@ -103,14 +103,14 @@ fails on the unwanted action and its own manual null check. A rubric judge must tell good answers from degraded ones before its scores count. It costs model calls; the sequential mode stops early. -`eval judge-validity` calibrates only dokimi's own answer judge: its answers +`eval judge validate` calibrates only dokimi's own answer judge: its answers file needs a `question_id` and the question text per row. It stops with `missing question_id` (exit 2) on a task-rubric answers file, so a task's own rubric judge is not covered by this command. Check that judge by hand: run it on a known-good and a degraded answer you wrote, and have a person read the scores. ```bash -python cli.py eval judge-validity --answers --sequential --strict +python cli.py eval judge validate --answers --sequential --strict ``` Then have a person review a sample of judged cells against the rubric and diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 4133acaa6..23906470b 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -66,7 +66,7 @@ checks the eval set itself: that it parses, that each task's grader fails an untouched workspace and passes the reference solution, and that an LLM judge scores degraded answers lower. -![Three checks of an eval set: valid (dokimi repo validate), graded right (dokimi repo tasks check), judge checked (dokimi eval judge-validity)](assets/readme/eval-checks.png) +![Three checks of an eval set: valid (dokimi repo validate), graded right (dokimi eval check), judge checked (dokimi eval judge validate)](assets/readme/eval-checks.png) **Where it is going:** mapping each claim in a skill to the tasks that exercise it, so the claims no eval tests are listed next to the score. @@ -93,7 +93,7 @@ From the studies so far ([evidence](skill-novelty-evidence.md)): | `skill:` | a skill (`SKILL.md`) injected as context | | `profile:` | an agent persona system prompt | | `agent:` / `skill-agent:` | **agentic** use — the model gets a tool and *decides* whether to fetch docs / load the skill | -| `skill-catalog:` | **selection** — every skill in a catalog is offered (a copy under a `template/` or `templates/` directory, a byte-identical mirror that loads under the same skill name, and anything under a vendored or VCS directory such as `.git`, `node_modules` or `.venv` are left out); scored on which one loads (`catalog trigger-set` builds the queries) | +| `skill-catalog:` | **selection** — every skill in a catalog is offered (a copy under a `template/` or `templates/` directory, a byte-identical mirror that loads under the same skill name, and anything under a vendored or VCS directory such as `.git`, `node_modules` or `.venv` are left out); scored on which one loads (`eval triggers generate` builds the queries) | Two fine-print notes: @@ -351,7 +351,7 @@ code. partly implemented)* what a repository provides to be evaluated as a whole: `repo-eval.yaml`, `task.json`, the `grade` entry point and `grade.json`, harness profiles, and the guidance reach probe; and how to run its task - self-check, agent runs, report and arm comparison (`repo tasks check`, + self-check, agent runs, report and arm comparison (`eval check`, `repo run`, `repo report`, `repo compare`). Tasks and graders stay in the scanned repository. - [repo-evaluation-mergeability.md](repo-evaluation-mergeability.md) — *(proposed)* grading whether a maintainer would merge a test-passing change: diff --git a/dokimi/docs/adding-doc-source.md b/dokimi/docs/adding-doc-source.md index 43097ae61..289d76056 100644 --- a/dokimi/docs/adding-doc-source.md +++ b/dokimi/docs/adding-doc-source.md @@ -70,7 +70,7 @@ python cli.py evaluate --product oneTBB \ --doc-source okf:/path/to/onetbb-okf-bundle ``` -`--doc-source` is supported by `answers generate`, `questions generate`, and `evaluate`. +`--doc-source` is supported by `answers generate`, `eval questions generate`, and `evaluate`. --- diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 0d9d48e56..d3a8f6d45 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -13,22 +13,22 @@ SPDX-License-Identifier: Apache-2.0 # Answer-only eval formats Inspect AI, OpenAI evals, skill-creator `evals.json` and promptfoo sets run on -dokimi's answer track. There is no separate runner. `questions import` +dokimi's answer track. There is no separate runner. `eval import` translates a set into a questions.v2 file in which every question carries a `grading` block of format-neutral checks. The usual commands then answer it -(`answers generate`, `arms run`), and `eval reference` scores the answers +(`answers generate`, `arms run`), and `eval score reference` scores the answers against those checks. ```bash -python cli.py questions import evals/born_first.jsonl --grader registry/evals/born-first.yaml -o q.json +python cli.py eval import evals/born_first.jsonl --grader registry/evals/born-first.yaml -o q.json python cli.py answers generate --product born-first --questions q.json --doc-source none \ --model claude-haiku-5-5 --provider amazon-bedrock --output answers.json -python cli.py eval reference --answers answers.json --questions q.json -o eval.json +python cli.py eval score reference --answers answers.json --questions q.json -o eval.json ``` -`tasks inspect ` names the format, and `questions import` uses the same +`eval inspect ` names the format, and `eval import` uses the same detection. Use `--format` to override it. A file that fits several formats -(`tasks inspect` lists the others under `alternatives`) is refused until +(`eval inspect` lists the others under `alternatives`) is refused until `--format` names one. ## The canonical record @@ -37,7 +37,7 @@ detection. Use `--format` to override it. A file that fits several formats |---|---| | `question` | The prompt as the native tool sends it. Chat inputs are flattened to one prompt, with `[role]` headings when there is more than one turn. MCQ rows are rendered with Inspect's single-answer template, or its multiple-answer one under `multiple_choice(multiple_correct=True)`. | | `prompt_style: verbatim` | The answerer sends `question` as it is, without its expert-answer wrapper, so that `ANSWER: X` and exact-match graders see a reply to the original prompt. A context arm, in `answers generate` or `arms run`, puts the docs first and the prompt after them. A multi-turn session arm still wraps its opening turn. | -| `expected_topics` | The reference answers, assertion values or expectations. `eval checklist` and the judge read them. | +| `expected_topics` | The reference answers, assertion values or expectations. `eval score checklist` and the judge read them. | | `grading.checks` | One check per native grader or assertion (below). Optional fields: `threshold`, `native_scorer`, `source_format`. | | `expected_skill` | skill-creator only: the skill the `evals.json` belongs to (`skill_name`, else the `/evals/` directory), so the set also scores [skill selection](evaluating-treatments.md#skill-selection-does-the-agent-load-the-right-skill) on a `skill-catalog:` arm. | | `metadata` | Sample metadata, promptfoo vars and prompt label, and the skill-creator `files`. | @@ -65,7 +65,7 @@ A blank value is not a check. A `match` value that is blank once the check has normalised it (stripped, and with `ignore_punctuation` on, `...` too), or an empty `regex` pattern, is found in every answer, so a row whose `ideal`/`target` is blank imports as `unsupported`, and a hand-written `values` -list with such an entry is refused by `eval reference`. An Inspect `exact` +list with such an entry is refused by `eval score reference`. An Inspect `exact` target that SQuAD-normalises to nothing (`...`, `the`) equals no answer, so it is dropped, and a sample left with no target imports as `unsupported`. A promptfoo assert whose value is blank (whitespace included) imports as @@ -93,7 +93,7 @@ count per arm as `errors`. An empty or content-less reply is an error, never gra it leaves the pass rate and the mean out, and is never cached (on store or replay), so a rerun asks the model again. -`eval reference` writes eval.v1 for an `answers generate` file and an arms +`eval score reference` writes eval.v1 for an `answers generate` file and an arms artifact for an `arms run` file. Each block carries `scorer: reference`, and the run's pass rates and mean scores are under `run_metadata.summary` (or `reference_scoring`). @@ -156,10 +156,10 @@ python cli.py eval native-import out.json --answers-out native_answers.json --ev The completions become an answers.v1 file. They are stored as `without_docs`, because the native run had no dokimi doc context. The tool's own verdicts -become an eval.v1 file with `scorer: native:`. Running `eval reference` +become an eval.v1 file with `scorer: native:`. Running `eval score reference` on that answers file re-grades the same completions with dokimi's checks, which shows where the two graders disagree. An Inspect sample that errored -keeps its error on the answer, so `eval reference` counts it as an error, not a +keeps its error on the answer, so `eval score reference` counts it as an error, not a score, as Inspect does. A promptfoo output must come from a single provider. Supported log formats are Inspect `json` logs (inspect-ai 0.3.283) and promptfoo `-o *.json` (0.125.0). diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 5aafd5a88..1b6750e00 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -347,7 +347,7 @@ next release. what its `--task` selection keeps; `dokimi/repos/compare_reach.py` how small an effect a row can find (`mde`, an uncorrected t-based estimate, and the lowest `p_holm` Holm's family leaves it). -- `dokimi/repos/selfcheck.py` — `repo tasks check`: on each task +- `dokimi/repos/selfcheck.py` — `eval check`: on each task `grade` must fail the untouched workspace and pass it after `oracle`, and an isolation probe must not see the tasks or the source from the agent's container. @@ -357,7 +357,7 @@ next release. `addressed` from the PR's own later commits), read with git and no GitHub call. It is the one command that writes into a repository's tasks directory. -Entry points: `python cli.py repo {validate,scan,fetch,reach,budget,tasks check,tasks replay,run,report}` +Entry points: `python cli.py repo {validate,scan,fetch,reach,budget,tasks replay,run,report}` (`python cli.py repo --help` is the current list). The interface a repository codes against is [repo-evaluation-contract.md](repo-evaluation-contract.md), still marked PROPOSED: the per-host egress proxy (§2) is not implemented, so diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index db3f19ccd..6749d8f4d 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -78,19 +78,19 @@ context artifact improve answers?" ### Default models Every command takes its models from `dokimi/defaults.py`, and every surface -(`benchmark`, `answers`, `eval`, `arms`, `subjects`, `evaluate`, `questions`, -`personas`, `novelty`) uses the same three: +(`benchmark`, `answers`, `eval`, `arms`, `subjects`, `evaluate`, `eval questions`, +`eval personas`, `novelty`) uses the same three: | Role | Default | List price, input/output per MTok | | --- | --- | --- | | Answer model (the model under test) | `openai/gpt-6.1-sol` | $2 / $10 | -| Judge (also RAGAS, `eval panel-score`, novelty extractor and retention adjudicator) | `anthropic/claude-sonnet-5-5` | $2 / $10 | +| Judge (also RAGAS, `eval score panel`, novelty extractor and retention adjudicator) | `anthropic/claude-sonnet-5-5` | $2 / $10 | | Generator (personas, questions, validation, panel review, gap fill) | `openai/gpt-6.1-sol` | $2 / $10 | The judge is a different family from the answerer, so the judge-independence warning stays quiet on default flags; answering with `claude-sonnet-5-5` moves the -novelty judge to `openai/gpt-6.1-sol` and makes `benchmark`, `eval score`, -`eval panel-score` and `evaluate` warn (`arms` and `subjects` do not check). Escalation +novelty judge to `openai/gpt-6.1-sol` and makes `benchmark`, `eval score pointwise`, +`eval score panel` and `evaluate` warn (`arms` and `subjects` do not check). Escalation judges (`--escalate-model`) have no default; `claude-opus-5-5` ($4 / $20) is the natural tie-break above the Sonnet judge. On `--provider amazon-bedrock` the bare names `claude-sonnet-5-5`, `claude-opus-5-5` and `claude-haiku-5-5` resolve to the @@ -112,7 +112,7 @@ checked), and on any other provider they exit before the first call and ask for **Defaults changed on 2026-10-13: results are not comparable with older baselines; re-baseline.** Before, `benchmark`/`answers`/`eval` answered with `gpt-5.5` and judged with `claude-sonnet-4-5-20250929`, while `arms`, `subjects`, `evaluate`, -`questions`, `personas` and `novelty` answered, judged and generated with an +`eval questions`, `eval personas` and `novelty` answered, judged and generated with an obsolete small OpenAI model. A score, delta or cost produced with default flags before that date is not comparable with one produced after: re-run the baseline (the same question set via `--questions-from`, the same arms) under the new defaults, or pin the old @@ -232,7 +232,7 @@ python cli.py answers generate \ --provider openai \ --debug-retrieval -python cli.py eval score \ +python cli.py eval score pointwise \ --product oneTBB \ --answers results/onetbb_golden/answers/onetbb.json \ --output results/onetbb_golden/eval/onetbb.json \ @@ -290,7 +290,7 @@ guidance part keeps the repository's guidance (`guidance:raw`). ```bash dokimi repo run @ --model --model --batch skills \ --arm guidance:raw,guidance:raw+skill:skills/lint,guidance:none+skill-catalog:skills -dokimi repo tasks check @ --arm skill-catalog:skills # paths exist, catalog has expected_skill +dokimi eval check @ --arm skill-catalog:skills # paths exist, catalog has expected_skill ``` When a cell installs skills, its `result.json` gets a `skills` object: @@ -383,12 +383,12 @@ pinned, only measured: run the same configuration several times `run_config`. Omitted from the request when unset. - `--judge-temperature`: sampling for the judge, recorded as `judge_temperature`. Applies to a single judge and to every role of a panel. - `eval judge-validity --judge-temperature` certifies the judge under the same - setting, and records it in the validity report. `eval score`, `eval - panel-score` (every panel judge) and `evaluate` take the same flag and record - it as `judge_temperature` in their output. So do `eval pairwise` (and its - escalation judge), `eval checklist`, `eval judge-wording`, `eval cross-judge` - (every `--judges` entry) and `eval judge-validity --mode pairwise`; each + `eval judge validate --judge-temperature` certifies the judge under the same + setting, and records it in the validity report. `eval score pointwise`, `eval + score panel` (every panel judge) and `evaluate` take the same flag and record + it as `judge_temperature` in their output. So do `eval score pairwise` (and its + escalation judge), `eval score checklist`, `eval judge wording`, `eval judge agreement` + (every `--judges` entry) and `eval judge validate --mode pairwise`; each probes every judge model with the setting first, before any judgement: a judge the provider table refuses exits 2 with no call made, and a probe call that the judge rejects exits 1, as `arms run` does. @@ -452,7 +452,7 @@ does not say another judge would agree, and on the perf golden set it did not: the same skill-vs-baseline answers scored +1.7 under the stored judge and −3.4 under Haiku 4.5 (both significant). Four commands make that visible. -**1. Gate the judge first.** `eval judge-validity` checks that the judge orders +**1. Gate the judge first.** `eval judge validate` checks that the judge orders answers whose order is known (an answer above its own damaged copy). `--sequential` adds bases until each degradation is decided instead of stopping at a fixed 20, which is usually inconclusive: across five seeds the fixed gate @@ -477,7 +477,7 @@ and rows keep pair order. A throttled call that still fails becomes an errored pair, which counts as a miss, so lower it when the summary shows errors. ```bash -python cli.py eval judge-validity --answers run.json --sequential \ +python cli.py eval judge validate --answers run.json --sequential \ --judge-provider amazon-bedrock --judge-model openai.gpt-oss-120b-1:0 \ --mode pairwise --output validity_pairwise.json ``` @@ -500,12 +500,12 @@ escalated). A gate pass is a floor, not a guarantee: gpt-oss was order-consistent on 95% of gate pairs but on only 44% of real skill-vs-baseline comparisons, which are much closer calls. -**2. Compare arms the same way.** `eval pairwise` puts each arm's answer next +**2. Compare arms the same way.** `eval score pairwise` puts each arm's answer next to the baseline's; `--escalate-model` is the same cheap-then-expensive cascade. -**3. Check the verdict across judges.** `eval cross-judge` re-scores an arms run +**3. Check the verdict across judges.** `eval judge agreement` re-scores an arms run under several judges (absolute, and with `--grounded` against each question's -`expected_topics`), takes `eval pairwise` and `eval checklist` reports as more +`expected_topics`), takes `eval score pairwise` and `eval score checklist` reports as more measures, and labels each arm `CONSISTENT ±`, `LEANING ±`, `JUDGE-DEPENDENT` or `NO DETECTABLE EFFECT`, on Holm-adjusted p over the measures that vote, or `NOT MEASURED` when no voting measure compared a single question (every call @@ -513,7 +513,7 @@ errored, or every judge was gated out); `--strict` exits 1 on that too. A `--judges` judge that leaves answers unscored prints how many and its first error, which the report keeps under `unscored`; one that scored none of its answers exits 2 once the report is written, with or without `--strict`, as -`eval pairwise` does when every comparison fails. With +`eval score pairwise` does when every comparison fails. With `--validity` reports, a judge that failed its gate is shown but does not vote (its grounded and checklist measures included, in any context); `--require-validated` leaves only judges that passed. The stored measure is @@ -528,8 +528,8 @@ judge sees what the stored judge saw. A file written before `judge_materials` ha only the report snippets; with stored scores on such material, `--judges` refuses (exit 2), and `--pairwise` or `--checklist` always do, since those judges see only the answers. `--no-stored` compares those judges with each other. The gate itself -passes no context by default (`eval pairwise` and `eval checklist` never pass -one), so it validates a judge in the no-context setting; `judge-validity --context +passes no context by default (`eval score pairwise` and `eval score checklist` never pass +one), so it validates a judge in the no-context setting; `eval judge validate --context stored` on an arms artifact gates it with each arm's stored material instead. That run exits 2 with a typed scorer or `--mode pairwise`, on a file with no non-empty verified material, when an arm loses all its bases, has only `padding` pairs or gets none sampled by @@ -537,16 +537,16 @@ verified material, when an arm loses all its bases, has only `padding` pairs or scored, or when no pair is usable. A label whose significant measures all rest on one judge says `(one judge)`; an escalated pairwise measure counts as its cheap judge's. Gate a judge on answers from the -same answer model it will judge: `cross-judge` warns when the validity report was +same answer model it will judge: `eval judge agreement` warns when the validity report was measured on another model's answers. ```bash -python cli.py eval cross-judge --answers run.json --questions golden.json --grounded --both \ +python cli.py eval judge agreement --answers run.json --questions golden.json --grounded --both \ --judges amazon-bedrock/us.anthropic.claude-haiku-5-5,amazon-bedrock/openai.gpt-oss-120b-1:0 \ --pairwise run_pairwise.json --validity validity_haiku.json --validity validity_pairwise.json ``` -**4. When changing the judge prompt**, `eval judge-wording` re-runs the gate +**4. When changing the judge prompt**, `eval judge wording` re-runs the gate under meaning-preserving rewrites of it and flags a degradation FRAGILE when a rewrite moves it on the same pairs: a paired sign test against the original prompt, Holm-adjusted over every rewrite and variant. Re-draws of the original @@ -555,7 +555,7 @@ scored pair (every call errored) is reported NOT MEASURED, not as "no rewrite moves anything", and `--strict` exits 1 on it. Use at least 30 bases; at 20 bases only a large wording effect is detectable. -`eval checklist` asks, per expected topic and in both polarities, whether the +`eval score checklist` asks, per expected topic and in both polarities, whether the answer covers it. Its sign was stable across judges, but it is the weakest of the measures (p ≈ 0.05 where pairwise reached 0.002) and costs about 2.7× an absolute judgement (Haiku, measured: $0.015 vs $0.0055 per answer); its value is @@ -797,7 +797,7 @@ A custom dotted name (`alice.model`) is not a vendor prefix, and a `huggingface/ A dated snapshot and its undated alias are neither: the alias does not record which snapshot answered, so the role is named on stderr as not verified, and `--strict` exits 1 on it. A baseline saved before a default-model change is such a baseline. A panel -artifact (`eval panel-score`) is compared by its judge models: the panels must have the same number of judges +artifact (`eval score panel`) is compared by its judge models: the panels must have the same number of judges and the same models, and a judge that matches the other panel's only through such an alias makes the role not verified. A role either side does not record (an older artifact, or answers with no model) is named on diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index f673acf55..fed99758f 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -21,7 +21,7 @@ pass rate against a baseline harness. ## Task Layouts Two layouts are read, and a root may mix them. `task.md` is read by -`tasks list`, `coverage` and `catalog scan`; it is run by `docker-oracle` and +`eval tasks list`, `coverage` and `catalog scan`; it is run by `docker-oracle` and by the `benchflow*:` harnesses (BenchFlow reads it natively), and a `command:` harness runs whatever it is given. What each harness refuses follows the files it reads, and a refused cell is an error (missing data), @@ -128,9 +128,9 @@ checks live links), and the six Harbor's SWE-bench adapter lists as failing unde oracle (harbor 6099b8c, `adapters/swebench/README.md`, Known Issues). Naming one in `--tasks` still runs it. -### Other eval formats: `tasks inspect` +### Other eval formats: `eval inspect` -`python cli.py tasks inspect [--json] [--out FILE]` names the eval-task format of a +`python cli.py eval inspect [--json] [--out FILE]` names the eval-task format of a file, a task directory, or every item under a tree, and reports its structure: kind, item count, which parts are present or missing (instruction, environment, oracle, verifier, metadata), lint problems, and what in dokimi @@ -159,14 +159,14 @@ real files in each format, with their sources in its `MANIFEST.yaml`. | `task-md` | directory with a BenchFlow `task.md` | executable | `benchflow*`, `docker-*` (with `environment/Dockerfile`) | | `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | `docker-*` run it as it is; `harbor`, `benchflow` through a cached Harbor export (not a `solution.yaml` task) | | `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | `docker-*` via `tasks run --swe-bench` (Python SWE-bench repos) | -| `inspect-samples` | JSON/JSONL/CSV rows with `input` and `target` (or the keys a `FieldSpec` names) | answer-only | `questions import` ([answer-only formats](answer-formats.md)) | -| `inspect-task` | `.py` importing `inspect_ai` with a `@task` function | answer-only (executable with a `sandbox=`) | `questions import` ([answer-only formats](answer-formats.md)) | -| `openai-evals` | JSONL rows with `input` (chat messages) and `ideal` | answer-only | `questions import` ([answer-only formats](answer-formats.md)) | +| `inspect-samples` | JSON/JSONL/CSV rows with `input` and `target` (or the keys a `FieldSpec` names) | answer-only | `eval import` ([answer-only formats](answer-formats.md)) | +| `inspect-task` | `.py` importing `inspect_ai` with a `@task` function | answer-only (executable with a `sandbox=`) | `eval import` ([answer-only formats](answer-formats.md)) | +| `openai-evals` | JSONL rows with `input` (chat messages) and `ideal` | answer-only | `eval import` ([answer-only formats](answer-formats.md)) | | `openai-evals-spec` | registry YAML with `class: evals.elsuite.*` run specs | answer-only | none yet | -| `skill-creator-evals` | `evals.json` with `skill_name` and `evals[].prompt` | answer-only | `questions import` ([answer-only formats](answer-formats.md)) | -| `promptfoo` | `promptfooconfig*.yaml`/`.json`, or a config with `prompts`, `providers`/`targets` and `tests`/`scenarios`/`redteam` | answer-only | `questions import` ([answer-only formats](answer-formats.md)) | -| `skill-creator-trigger` | rows with `query` and `should_trigger` | trigger | `catalog trigger-set` | -| `skill-reach` | `QuerySet` (`queries[]` with `query_id`/`id`, `text`, `expected_skill`) | trigger | `catalog trigger-set` | +| `skill-creator-evals` | `evals.json` with `skill_name` and `evals[].prompt` | answer-only | `eval import` ([answer-only formats](answer-formats.md)) | +| `promptfoo` | `promptfooconfig*.yaml`/`.json`, or a config with `prompts`, `providers`/`targets` and `tests`/`scenarios`/`redteam` | answer-only | `eval import` ([answer-only formats](answer-formats.md)) | +| `skill-creator-trigger` | rows with `query` and `should_trigger` | trigger | `eval triggers generate` | +| `skill-reach` | `QuerySet` (`queries[]` with `query_id`/`id`, `text`, `expected_skill`) | trigger | `eval triggers generate` | | `promptfoo-tests` | CSV with `__expected` columns, or rows of `{vars, assert}` | answer-only | none yet | | `dokimi-questions` | questions.v1/v2 JSON | answer-only (trigger with `expected_skill`) | questions track | | `dokimi-explore-task` | YAML with `id`, `query` and `references[].path` | localization | explore track | @@ -195,7 +195,7 @@ sets `deploy.resources.limits.memory`, which `docker compose` honours and dokimi reads as the task's own value. `DOKIMI_TBENCH_V1_MEMORY_MB` changes the 8192 default. The Harbor export writes the same value to `task.toml`. -`tasks list` and `tasks run` still read only the first three; `tasks run --swe-bench` +`eval tasks list` and `tasks run` still read only the first three; `tasks run --swe-bench` also takes a SWE-bench file. The `docker-*` harnesses mount the oracle at `/solution` and `/oracle` and the diff --git a/dokimi/docs/contributing-terminal-bench-task.md b/dokimi/docs/contributing-terminal-bench-task.md index 35fbeb04d..a72f2088b 100644 --- a/dokimi/docs/contributing-terminal-bench-task.md +++ b/dokimi/docs/contributing-terminal-bench-task.md @@ -19,13 +19,13 @@ Task definitions are evaluation content, so they live in a separate content repository, not here. This framework runs them from a checkout you name: ```bash -python cli.py tasks list --tasks-root /evaluation/harbor/tasks +python cli.py eval tasks list --tasks-root /evaluation/harbor/tasks python cli.py tasks run --tasks-root /evaluation/harbor/tasks \ --tasks --harnesses docker-oracle --out-json results/runs.json ``` This page is the format reference: everything a task directory must contain for -`tasks list`/`tasks run`, the `docker-*` harnesses and the coverage contract to +`eval tasks list`/`tasks run`, the `docker-*` harnesses and the coverage contract to read it. Contribute the task itself to the content repository; contribute changes to this format here, next to the code that parses it ([`dokimi/harnesses/tasks.py`](../dokimi/harnesses/tasks.py), @@ -108,8 +108,8 @@ Field by field, with the consumer that actually reads it: | Field | Read by | Meaning | | --- | --- | --- | -| `metadata.tags` | `tasks list`, the coverage contract | Lowercased tag list. **This is the work signal**: a task counts as work for a product when its tags contain the product's key in `products.yaml` (or one of that product's `tag_aliases:`). Include the library key (`onetbb`, `dpnp`, `intel-performance-skills`), the concept (`parallel-reduce`), and any provenance tag (`parres`). | -| `metadata.difficulty` | `tasks list`, the task-runs report | `easy` / `medium` / `hard`; drives the difficulty rollup in the Markdown report. Missing renders as `?`. | +| `metadata.tags` | `eval tasks list`, the coverage contract | Lowercased tag list. **This is the work signal**: a task counts as work for a product when its tags contain the product's key in `products.yaml` (or one of that product's `tag_aliases:`). Include the library key (`onetbb`, `dpnp`, `intel-performance-skills`), the concept (`parallel-reduce`), and any provenance tag (`parres`). | +| `metadata.difficulty` | `eval tasks list`, the task-runs report | `easy` / `medium` / `hard`; drives the difficulty rollup in the Markdown report. Missing renders as `?`. | | `metadata.level` | [`difficulty-rubric.md`](./difficulty-rubric.md) | Workflow depth: `triage`, `diagnosis`, `end_to_end`. Optional but recommended for skill evaluation. | | `metadata.difficulty_explanation`, `category`, `author_*` | humans | Provenance and review context; carried through in the run artifact. | | `agent.timeout_sec` | `TaskSpec.timeout_sec`, the solver harnesses | Wall-clock budget for the agent's attempt. Typically 300. | @@ -250,7 +250,7 @@ tells you nothing about whether the verifier discriminates. Also confirm the task is discoverable and tagged as intended: ```bash -python cli.py tasks list --tasks-root /evaluation/harbor/tasks +python cli.py eval tasks list --tasks-root /evaluation/harbor/tasks python -m dokimi.coverage \ --tasks-root /evaluation/harbor/tasks \ --questions-root /questions diff --git a/dokimi/docs/difficulty-rubric.md b/dokimi/docs/difficulty-rubric.md index e242f3901..e5266dcc6 100644 --- a/dokimi/docs/difficulty-rubric.md +++ b/dokimi/docs/difficulty-rubric.md @@ -59,7 +59,7 @@ sibling — and route straight to the matching pattern. See the trigger axis bel Orthogonal to level and difficulty. Tag each item with the kind of input that begins the task, recorded as `metadata.trigger`. -Not to be confused with *trigger sets* (`catalog trigger-set`, see +Not to be confused with *trigger sets* (`eval triggers generate`, see [Skill selection](evaluating-treatments.md#skill-selection-does-the-agent-load-the-right-skill)). Those label which skill a message should load and measure selection. This axis tags what input starts the work once the skill is in play. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 7219d8f44..bbb35e898 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -63,7 +63,7 @@ given `--judge`. For docs arms, the material is the retrieved chunks, rarely identical across questions, so each question adds its own copy (up to the 15,000-char cut); on a synthetic 20-question docs run the artifact grew from 93,209 to 392,852 bytes (4.2x). Sharing an arms artifact shares the skill or doc -text it contains. With `cross-judge --judges`, the judge is given the same text +text it contains. With `eval judge agreement --judges`, the judge is given the same text through `JudgeContextStore.from_artifact`, which skips any text that does not match its digest. Question ids must be unique for this (a question with no id is given a positional one; duplicated ids are refused). @@ -84,7 +84,7 @@ judge, 24 bases, one run), the full material raised the mean intact score from 0.674, not significant at this size), unchanged on wrong numbers. A skill arm's delta measured with the material is therefore on a higher level than one measured without it, and the difference between the two is not evidence that -the skill helps. `eval judge-validity` passes no context by default; run it on an +the skill helps. `eval judge validate` passes no context by default; run it on an arms artifact with `--context stored` to gate the judge the way it scored the arms — each answer with its arm's stored material (bases with no stored entry are dropped and counted, except baseline with the no-context marker). It exits 2 @@ -517,14 +517,14 @@ jq '[.questions[] | select(.expected_skill == "linux-perf") | .question]' trigge *Trigger set* here means a selection query set: messages labelled with the skill that should load. It is unrelated to the `trigger` axis of the [difficulty rubric](difficulty-rubric.md#axis-0--trigger-what-starts-the-work), which tags -what input starts a task (`counter`, `source`, `new_code`). `catalog trigger-set` does not +what input starts a task (`counter`, `source`, `new_code`). `eval triggers generate` does not carry `metadata.trigger` into its output. -**Build the set.** `catalog trigger-set` labels question files, lints them, and writes +**Build the set.** `eval triggers generate` labels question files, lints them, and writes the format a selection tool reads: ```bash -python cli.py catalog trigger-set --catalog /skills \ +python cli.py eval triggers generate --catalog /skills \ --questions /questions/intel_performance_skills_golden.json=@metadata.skill_under_test \ --questions /questions/onetbb_golden.json=onetbb-quickstart \ --questions /questions/onedal_golden.json=none \ @@ -550,7 +550,7 @@ python cli.py catalog trigger-set --catalog /skills \ rest as `acceptable_skills`, which skill-reach treats as neutral helpers rather than hits, so on those queries it is stricter than the `skill-catalog:` arm; `skill-creator --target ` (a catalog skill) writes an [anthropics/skills skill-creator](https://github.com/anthropics/skills/tree/main/skills/skill-creator) - eval set; `questions` writes `questions.v2` with `expected_skill` for the arm below. + eval set; `eval triggers generate --format questions` writes `questions.v2` with `expected_skill` for the arm below. **Proxy: the `skill-catalog:` arm.** Runs in the normal arms loop on any model, and the report gains a *Skill selection* table (accuracy, recall, recall (any), multi-load, diff --git a/dokimi/docs/external-content.md b/dokimi/docs/external-content.md index 4eb88f845..5ee4ae9be 100644 --- a/dokimi/docs/external-content.md +++ b/dokimi/docs/external-content.md @@ -23,7 +23,7 @@ implicit default that points at a sibling checkout: |---|---|---| | `--root` | a whole content checkout to scan | `catalog scan` | | `--questions-root` | directory holding `_golden.json` | `benchmark preflight/run/batch`, `evaluate`, `coverage`, `config_check` | -| `--tasks-root` | directory of executable task dirs (each with `task.toml` or a BenchFlow `task.md`), or of exploration task YAML files | `tasks list/run`, `explore list/run`, `coverage`, `config_check` | +| `--tasks-root` | directory of executable task dirs (each with `task.toml` or a BenchFlow `task.md`), or of exploration task YAML files | `eval tasks list`/`tasks run`, `explore list/run`, `coverage`, `config_check` | | `--repo-root` | the source tree an exploration task is scored against | `explore run` | | `--registry` | a `products.yaml` describing products this repo has never heard of | `benchmark preflight/run` | | `--questions` | one question-set file | `arms run`, `answers generate` | @@ -234,7 +234,7 @@ report. Section 5 shows how to check the arm specs resolve first. `--tasks-root` is the directory that *contains* task directories: ```console -$ python cli.py tasks list --tasks-root C:/Users/Public/content-repo/evaluation/harbor/tasks +$ python cli.py eval tasks list --tasks-root C:/Users/Public/content-repo/evaluation/harbor/tasks demo-widget easy demo,widget onetbb-parallel-reduce medium onetbb ``` @@ -391,16 +391,16 @@ Worth knowing before you read a suspiciously clean result: | Command | Root missing | Exit | |---|---|---| -| `catalog scan --root ` | `--root is not a directory: ` | 1 | -| `tasks run --tasks-root ` | `FileNotFoundError: Task directory not found: demo-widget` | 1 | -| `tasks list --tasks-root ` | prints nothing | 0 | +| `catalog scan --root ` | `--root is not a directory: ` | 2 | +| `tasks run --tasks-root ` | `--tasks-root is not a directory: ` | 2 | +| `eval tasks list --tasks-root ` | `--tasks-root is not a directory: ` | 2 | | `explore list --tasks-root ` | `--tasks-root is not a directory: ` | 2 | | `coverage` with no roots | every product reads `no`/`0`, plus an explicit note | 0 | | `evaluate`/`benchmark run` with a root lacking the golden set | `WARNING ... has no golden set for ` and continues | — | -`tasks list` is the trap: an empty listing and a correct listing both exit 0, -so a typo in a path looks like an empty checkout. `explore list` refuses an -empty root with exit 2. The `coverage` matrix is explicit about it instead: +`eval tasks list` is the trap for an existing directory: an empty listing and a correct listing +both exit 0, so an empty checkout looks like a correct one. A missing root is refused with exit 2, +not a silent success. `explore list` refuses an empty root with exit 2. The `coverage` matrix is explicit about it instead: ```console $ python -m dokimi.coverage diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md index a8e6259b0..dc5821aba 100644 --- a/dokimi/docs/methodology.md +++ b/dokimi/docs/methodology.md @@ -85,7 +85,7 @@ The benchmark separates model responsibilities because self-judging can inflate The defaults live in `dokimi/defaults.py` and changed on 2026-10-13; scores from earlier default-flag runs are not comparable with later ones — re-baseline ([Default models](benchmarking-and-comparison.md#default-models)). -`benchmark`, `eval score`, `eval panel-score` and `evaluate` warn when the normalized answer model and judge model match (`arms` and `subjects` do not check). Evaluation artifacts also store model/provider metadata and `question_set_hash` so later comparisons can prove they used the same question set. +`benchmark`, `eval score pointwise`, `eval score panel` and `evaluate` warn when the normalized answer model and judge model match (`arms` and `subjects` do not check). Evaluation artifacts also store model/provider metadata and `question_set_hash` so later comparisons can prove they used the same question set. Supported providers in the main pipeline include OpenAI, Anthropic, Google Vertex, OpenRouter, Amazon Bedrock, and OpenAI Codex where wired by the CLI. @@ -108,9 +108,9 @@ A question set is a JSON object with a `questions` list. The framework writes it - **v2 (written):** each record needs `id` and a non-empty `question`. A v2 record may not also carry `text`, so no record holds two question texts that readers could disagree on. - **v1 (still read):** the same record with the text under `text`. Curated golden sets in content repositories are v1, stamped `"schema_version": "questions.v1"` or unstamped; an unstamped file is read as whichever version it matches. -Every writer (`questions generate`, `questions refine`, the `evaluate`/`benchmark` pipeline's `questions/.json`) renames `text` to `question` and validates the set against v2 before writing it; a set that fails is kept at `.rejected`, before any answer is generated. The rename is the only change between the versions: `difficulty`, `metadata` and every other field are carried over as written, except a `null` the reader drops (below). New curated sets should use `question`. Records may carry fields the schema does not declare (writers pass curated records through verbatim), but the declared ones are type-checked. +Every writer (`eval questions generate`, `eval questions refine`, the `evaluate`/`benchmark` pipeline's `questions/.json`) renames `text` to `question` and validates the set against v2 before writing it; a set that fails is kept at `.rejected`, before any answer is generated. The rename is the only change between the versions: `difficulty`, `metadata` and every other field are carried over as written, except a `null` the reader drops (below). New curated sets should use `question`. Records may carry fields the schema does not declare (writers pass curated records through verbatim), but the declared ones are type-checked. -**Reading is tolerant; input files are never rewritten by a read operation.** Every command that reads a question set (the pipeline's cached `questions/.json`, `--questions-from`, `--custom-questions` and the auto-included golden set, `arms run`, `answers generate`, `report generate`, `questions refine`, `eval checklist`, `dataset export`) reads it through one function, `dokimi.questions.artifact.load_questions`. It accepts v1, v2, and the older shapes earlier releases wrote or passed through, and works on the set as v2 in memory: +**Reading is tolerant; input files are never rewritten by a read operation.** Every command that reads a question set (the pipeline's cached `questions/.json`, `--questions-from`, `--custom-questions` and the auto-included golden set, `arms run`, `answers generate`, `report generate`, `eval questions refine`, `eval score checklist`, `dataset export`) reads it through one function, `dokimi.questions.artifact.load_questions`. It accepts v1, v2, and the older shapes earlier releases wrote or passed through, and works on the set as v2 in memory: | Read as | becomes | |---|---| @@ -121,7 +121,7 @@ Every writer (`questions generate`, `questions refine`, the `evaluate`/`benchmar | `metadata` as a list | `{"items": [...]}` (v2 declares `metadata` an object; the list is kept, not dropped) | | `null` in a key v1 does not declare and v2 types (`model`, `total_questions`, `persona_id`, `source_type`, `topics`, ...) | key dropped, without a notice. A `null` inside a typed value (`"topics": [null]`) is not dropped and is refused | -A set that needed any of these, or that is not valid under the version it declares (e.g. a set stamped `questions.v1` whose records use `question`, which `questions refine` wrote before v2), is logged once: `read in legacy questions format (...); using it as questions.v2`. Reading these shapes is meant to be retired later; fix the file by hand to stop the notice (`question` for `text` or `question_text`, an `id` on each record, `metadata` as an object, and `schema_version` set to `questions.v2` or removed). No command rewrites a set unchanged: `questions refine` is not a re-save, it normalizes fields, drops trivial and duplicate questions and changes the `question_set_hash`, so do not use it on a curated set. A set that is still not valid v2 after reading (a record with an empty question, a field of the wrong type) fails before any model call, with each failing record and the fix named. The run then writes its own v2 copy to `/questions/.json` with the `question_set_hash`. A `--questions-from` file is never written. A `--custom-questions` file is written only through the Python API (`EvaluationPipeline(custom_questions_path=/questions/.json, force_regen=True)`), when the merge added a record to it; that file keeps each record's `source_type`, while any other custom or golden set is tagged `manual`. The cached file is the run's own and is rewritten as v2. No CLI command can reach this path: `evaluate` takes `--custom-questions` but not `--force-regen`, and `benchmark run` and `benchmark batch` take `--force-regen` but not `--custom-questions`. For sets already valid as v1 or v2, ids and `question_set_hash` do not change on the way through. `dataset export --kind questions` exports any of these in the v2 shape and records the source version (or `legacy`) in the dataset card. +A set that needed any of these, or that is not valid under the version it declares (e.g. a set stamped `questions.v1` whose records use `question`, which `eval questions refine` wrote before v2), is logged once: `read in legacy questions format (...); using it as questions.v2`. Reading these shapes is meant to be retired later; fix the file by hand to stop the notice (`question` for `text` or `question_text`, an `id` on each record, `metadata` as an object, and `schema_version` set to `questions.v2` or removed). No command rewrites a set unchanged: `eval questions refine` is not a re-save, it normalizes fields, drops trivial and duplicate questions and changes the `question_set_hash`, so do not use it on a curated set. A set that is still not valid v2 after reading (a record with an empty question, a field of the wrong type) fails before any model call, with each failing record and the fix named. The run then writes its own v2 copy to `/questions/.json` with the `question_set_hash`. A `--questions-from` file is never written. A `--custom-questions` file is written only through the Python API (`EvaluationPipeline(custom_questions_path=/questions/.json, force_regen=True)`), when the merge added a record to it; that file keeps each record's `source_type`, while any other custom or golden set is tagged `manual`. The cached file is the run's own and is rewritten as v2. No CLI command can reach this path: `evaluate` takes `--custom-questions` but not `--force-regen`, and `benchmark run` and `benchmark batch` take `--force-regen` but not `--custom-questions`. For sets already valid as v1 or v2, ids and `question_set_hash` do not change on the way through. `dataset export --kind questions` exports any of these in the v2 shape and records the source version (or `legacy`) in the dataset card. `expected_topics` carries two different kinds of content depending on the source, and evaluation treats them differently. Persona-generated sets store short keyword labels ("version check", "import test"). Curated golden sets store verified answer statements, often with citations in `metadata` (`must_cite`, `github_issue`, `verified_by`). Only the latter are usable as judge ground truth — see [Grounded Judging](#grounded-judging). diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index b2dd290ed..4b0495433 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -71,7 +71,7 @@ All commands run from the repo root and use `python cli.py …`. Use ### Step 1 — Discover personas ```bash -python cli.py personas discover \ +python cli.py eval personas discover \ --product oneTBB \ --repo uxlfoundation/oneTBB \ --count 5 \ @@ -86,13 +86,13 @@ Review and approve: ```bash $EDITOR personas/oneTBB.json -python cli.py personas approve --file personas/oneTBB.json +python cli.py eval personas validate --file personas/oneTBB.json ``` ### Step 2 — Generate questions ```bash -python cli.py questions generate \ +python cli.py eval questions generate \ --product oneTBB \ --personas personas/oneTBB.json \ --count 2 \ @@ -127,7 +127,7 @@ alternative sources — see [adding-doc-source.md](adding-doc-source.md). ### Step 4 — Score answers ```bash -python cli.py eval score \ +python cli.py eval score pointwise \ --product oneTBB \ --answers answers/oneTBB.json \ --output eval/oneTBB.json \ @@ -142,7 +142,7 @@ computes an aggregate and the context-arm minus baseline delta. Multi-judge panel: ```bash -python cli.py eval panel-score \ +python cli.py eval score panel \ --product oneTBB \ --answers answers/oneTBB.json \ --output eval/oneTBB.panel.json @@ -183,11 +183,11 @@ both arms; a run that scored one arm only shows that arm's mean. PRODUCT=oneTBB REPO=uxlfoundation/oneTBB -python cli.py personas discover --product $PRODUCT --repo $REPO --count 5 && \ -python cli.py personas approve --file personas/$PRODUCT.json && \ -python cli.py questions generate --product $PRODUCT --personas personas/$PRODUCT.json --validate && \ +python cli.py eval personas discover --product $PRODUCT --repo $REPO --count 5 && \ +python cli.py eval personas validate --file personas/$PRODUCT.json && \ +python cli.py eval questions generate --product $PRODUCT --personas personas/$PRODUCT.json --validate && \ python cli.py answers generate --product $PRODUCT --questions questions/$PRODUCT.json --model gpt-6.1-sol && \ -python cli.py eval score --product $PRODUCT --answers answers/$PRODUCT.json --judge-model claude-sonnet-5-5 --judge-provider anthropic && \ +python cli.py eval score pointwise --product $PRODUCT --answers answers/$PRODUCT.json --judge-model claude-sonnet-5-5 --judge-provider anthropic && \ python cli.py report generate --product $PRODUCT --eval eval/$PRODUCT.json --questions questions/$PRODUCT.json --output reports/$PRODUCT.md ``` @@ -250,19 +250,19 @@ failed check. | Code | Meaning | Examples | |------|---------|----------| | 0 | The command ran and, if it checks something, the check passed. | | -| 1 | A verdict or a gate that ran and failed, or a runtime failure (a model call, a run, a clone or fetch). | `baseline compare --strict` over a changed model, `repo reach` with a harness binary that cannot run, `catalog trigger-set --strict` over a lint finding | +| 1 | A verdict or a gate that ran and failed, or a runtime failure (a model call, a run, a clone or fetch). | `baseline compare --strict` over a changed model, `repo reach` with a harness binary that cannot run, `eval triggers generate --strict` over a lint finding | | 2 | A bad invocation: a missing, unreadable or malformed input file, a flag combination the command refuses, an output file that cannot be written, a missing provider credential, a source string that is malformed, unpinned or not a repository root. | `baseline save` on a bad eval file, `benchmark preflight` with no golden set, `repo scan --manifest ../x.yaml` | A command that validates its input and then runs exits 2 for the first and 1 for the second. Not every command follows this yet: `repo run` and -`repo tasks check` still exit 1 for their input refusals, and `tools preference` +`eval check` still exit 1 for their input refusals, and `tools preference` exits 2 when every call failed or `--max-cost-usd` skipped calls. ## Troubleshooting | Symptom | Fix | | --- | --- | -| LLM calls fail / wrong provider | All providers go through LiteLLM (`dokimi/llm.py`); set the matching `*_API_KEY` and pass `--provider`/`--model`. LangChain is used only by `eval ragas`, which wraps its judge in `ChatOpenAI`/`ChatAnthropic`; it and ragas come with the `ragas` extra (`pip install 'dokimi[ragas]'`). | +| LLM calls fail / wrong provider | All providers go through LiteLLM (`dokimi/llm.py`); set the matching `*_API_KEY` and pass `--provider`/`--model`. LangChain is used only by `eval score ragas`, which wraps its judge in `ChatOpenAI`/`ChatAnthropic`; it and ragas come with the `ragas` extra (`pip install 'dokimi[ragas]'`). | | `No module named numpy` | Pipeline falls back to exact-text dedup; install numpy for embedding-based dedup | | `GitHub client not available` | Pipeline uses `gh` CLI fallback; `pip install PyGithub` to silence | | Context7 timeout | Raise `context7.timeout` in `dokimi/config/products.yaml` (or in a config/products.yaml of your own in the working directory) | diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 4f6136625..3bc5add74 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -2935,194 +2935,6 @@ "path": "dokimi catalog trigger-probe", "subcommand_required": null, "subcommands": {} - }, - "trigger-set": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--catalog", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'questions'", - "'reach'", - "'skill-creator'" - ], - "const": null, - "default": "reach", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--format", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--questions", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--rewrite", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--strict", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--target", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--trigger-set", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi catalog trigger-set", - "subcommand_required": null, - "subcommands": {} } } }, @@ -3873,7 +3685,7 @@ "path": "dokimi eval", "subcommand_required": true, "subcommands": { - "checklist": { + "check": { "allow_abbrev": true, "params": [ { @@ -3885,15 +3697,31 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--answers", - "kind": "option", + "key": "source", + "kind": "argument", "nargs": null, - "position": null, + "position": 0, "required": true, "type": null }, { - "action": "_StoreAction", + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--allow-unsupported", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", "choices": null, "const": null, "default": null, @@ -3901,7 +3729,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--arms", + "key": "--arm", "kind": "option", "nargs": null, "position": null, @@ -3917,7 +3745,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--baseline-arm", + "key": "--build-network", "kind": "option", "nargs": null, "position": null, @@ -3928,12 +3756,12 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 5, + "default": 10800, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--concurrency", + "key": "--build-timeout", "kind": "option", "nargs": null, "position": null, @@ -3941,17 +3769,17 @@ "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" }, { - "action": "_HelpAction", + "action": "_AppendAction", "choices": null, "const": null, - "default": "==SUPPRESS==", + "default": null, "group_key": null, "group_required": null, "hidden": false, - "is_flag": true, - "key": "--help/-h", + "is_flag": false, + "key": "--cache", "kind": "option", - "nargs": 0, + "nargs": null, "position": null, "required": false, "type": null @@ -3960,12 +3788,12 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": "claude-sonnet-5-5", + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--judge-model", + "key": "--cache-dir", "kind": "option", "nargs": null, "position": null, @@ -3974,22 +3802,14 @@ }, { "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], + "choices": null, "const": null, - "default": "anthropic", + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--judge-provider", + "key": "--cache-root", "kind": "option", "nargs": null, "position": null, @@ -4000,72 +3820,48 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": null, + "default": "docker", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--judge-temperature", + "key": "--docker", "kind": "option", "nargs": null, "position": null, "required": false, - "type": "float" + "type": null }, { "action": "_StoreAction", "choices": null, "const": null, - "default": null, + "default": 3600, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--output", + "key": "--grade-timeout", "kind": "option", "nargs": null, "position": null, "required": false, - "type": null + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" }, { - "action": "_StoreAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": null, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--questions", + "is_flag": true, + "key": "--help/-h", "kind": "option", - "nargs": null, + "nargs": 0, "position": null, - "required": true, - "type": null - } - ], - "path": "dokimi eval checklist", - "subcommand_required": null, - "subcommands": {} - }, - "coverage": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "evals", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, + "required": false, "type": null }, { @@ -4077,69 +3873,45 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--graph", + "key": "--image", "kind": "option", "nargs": null, "position": null, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, "required": false, "type": null }, { - "action": "_AppendAction", + "action": "_StoreAction", "choices": null, "const": null, - "default": null, + "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--min", + "key": "--jobs/-j", "kind": "option", "nargs": null, "position": null, "required": false, - "type": "dokimi.commands.eval_coverage._floor#bbd5737352a9" + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" }, { "action": "_StoreAction", "choices": null, "const": null, - "default": null, + "default": ".agent-eval/repo-eval.yaml", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--out", + "key": "--manifest", "kind": "option", "nargs": null, "position": null, "required": false, "type": null - } - ], - "path": "dokimi eval coverage", - "subcommand_required": null, - "subcommands": {} - }, - "cross-judge": { - "allow_abbrev": true, - "params": [ + }, { "action": "_StoreAction", "choices": null, @@ -4149,23 +3921,23 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--answers", + "key": "--manifest-file", "kind": "option", "nargs": null, "position": null, - "required": true, + "required": false, "type": null }, { "action": "_StoreAction", "choices": null, "const": null, - "default": null, + "default": "repo-check", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--baseline-arm", + "key": "--out", "kind": "option", "nargs": null, "position": null, @@ -4181,7 +3953,7 @@ "group_required": null, "hidden": false, "is_flag": true, - "key": "--both", + "key": "--rebuild-image", "kind": "option", "nargs": 0, "position": null, @@ -4189,7 +3961,7 @@ "type": null }, { - "action": "_AppendAction", + "action": "_StoreAction", "choices": null, "const": null, "default": null, @@ -4197,7 +3969,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--checklist", + "key": "--source-dir", "kind": "option", "nargs": null, "position": null, @@ -4205,20 +3977,20 @@ "type": null }, { - "action": "_StoreAction", + "action": "_AppendAction", "choices": null, "const": null, - "default": 5, + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--concurrency", + "key": "--task", "kind": "option", "nargs": null, "position": null, "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + "type": null }, { "action": "_StoreTrueAction", @@ -4229,27 +4001,35 @@ "group_required": null, "hidden": false, "is_flag": true, - "key": "--grounded", + "key": "--warm", "kind": "option", "nargs": 0, "position": null, "required": false, "type": null - }, + } + ], + "path": "dokimi eval check", + "subcommand_required": null, + "subcommands": {} + }, + "coverage": { + "allow_abbrev": true, + "params": [ { - "action": "_HelpAction", + "action": "_StoreAction", "choices": null, "const": null, - "default": "==SUPPRESS==", + "default": null, "group_key": null, "group_required": null, "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, + "is_flag": false, + "key": "evals", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, "type": null }, { @@ -4261,44 +4041,44 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--judge-temperature", + "key": "--graph", "kind": "option", "nargs": null, "position": null, - "required": false, - "type": "float" + "required": true, + "type": null }, { - "action": "_StoreAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": null, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--judges", + "is_flag": true, + "key": "--help/-h", "kind": "option", - "nargs": null, + "nargs": 0, "position": null, "required": false, "type": null }, { - "action": "_StoreTrueAction", + "action": "_AppendAction", "choices": null, - "const": true, - "default": false, + "const": null, + "default": null, "group_key": null, "group_required": null, "hidden": false, - "is_flag": true, - "key": "--no-stored", + "is_flag": false, + "key": "--min", "kind": "option", - "nargs": 0, + "nargs": null, "position": null, "required": false, - "type": null + "type": "dokimi.commands.eval_coverage._floor#bbd5737352a9" }, { "action": "_StoreAction", @@ -4309,15 +4089,23 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--output", + "key": "--out", "kind": "option", "nargs": null, "position": null, "required": false, "type": null - }, + } + ], + "path": "dokimi eval coverage", + "subcommand_required": null, + "subcommands": {} + }, + "import": { + "allow_abbrev": true, + "params": [ { - "action": "_AppendAction", + "action": "_StoreAction", "choices": null, "const": null, "default": null, @@ -4325,11 +4113,11 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--pairwise", - "kind": "option", + "key": "path", + "kind": "argument", "nargs": null, - "position": null, - "required": false, + "position": 0, + "required": true, "type": null }, { @@ -4341,7 +4129,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--product", + "key": "--format", "kind": "option", "nargs": null, "position": null, @@ -4357,7 +4145,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--questions", + "key": "--grader", "kind": "option", "nargs": null, "position": null, @@ -4365,15 +4153,15 @@ "type": null }, { - "action": "_StoreTrueAction", + "action": "_HelpAction", "choices": null, - "const": true, - "default": false, + "const": null, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, - "key": "--require-validated", + "key": "--help/-h", "kind": "option", "nargs": 0, "position": null, @@ -4381,23 +4169,39 @@ "type": null }, { - "action": "_StoreTrueAction", + "action": "_StoreAction", "choices": null, - "const": true, - "default": false, + "const": null, + "default": null, "group_key": null, "group_required": null, "hidden": false, - "is_flag": true, - "key": "--strict", + "is_flag": false, + "key": "--output/-o", "kind": "option", - "nargs": 0, + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--scorer", + "kind": "option", + "nargs": null, "position": null, "required": false, "type": null }, { - "action": "_AppendAction", + "action": "_StoreAction", "choices": null, "const": null, "default": null, @@ -4405,7 +4209,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--validity", + "key": "--task", "kind": "option", "nargs": null, "position": null, @@ -4413,11 +4217,11 @@ "type": null } ], - "path": "dokimi eval cross-judge", + "path": "dokimi eval import", "subcommand_required": null, "subcommands": {} }, - "grounding": { + "inspect": { "allow_abbrev": true, "params": [ { @@ -4429,10 +4233,10 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--answers", - "kind": "option", + "key": "path", + "kind": "argument", "nargs": null, - "position": null, + "position": 0, "required": true, "type": null }, @@ -4453,17 +4257,17 @@ "type": null }, { - "action": "_StoreAction", + "action": "_StoreTrueAction", "choices": null, - "const": null, - "default": null, + "const": true, + "default": false, "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--output", + "is_flag": true, + "key": "--json", "kind": "option", - "nargs": null, + "nargs": 0, "position": null, "required": false, "type": null @@ -4472,6848 +4276,985 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 0.5, + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--threshold", + "key": "--out", "kind": "option", "nargs": null, "position": null, "required": false, - "type": "float" + "type": null } ], - "path": "dokimi eval grounding", + "path": "dokimi eval inspect", "subcommand_required": null, "subcommands": {} }, - "judge-validity": { + "judge": { "allow_abbrev": true, "params": [ { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": 20, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--bases", + "is_flag": true, + "key": "--help/-h", "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": [ - "'none'", - "'stored'" - ], - "const": null, - "default": "none", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--context", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--escalate-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--escalate-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--escalate-ties", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-temperature", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 120, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-bases", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0.8, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--min-accuracy", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": [ - "'absolute'", - "'pairwise'" - ], - "const": null, - "default": "absolute", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--mode", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "judge", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--scorer", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--seed", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--sequential", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--strict", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--typed-max-len", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.evaluate._non_negative_int#95d3497f2297" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--typed-threads", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--variants", - "kind": "option", - "nargs": null, + "nargs": 0, "position": null, "required": false, "type": null } ], - "path": "dokimi eval judge-validity", - "subcommand_required": null, - "subcommands": {} - }, - "judge-wording": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 20, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--bases", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-temperature", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0.8, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--min-accuracy", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 1, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--redraws", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--seed", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--strict", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--variants", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--wordings", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi eval judge-wording", - "subcommand_required": null, - "subcommands": {} - }, - "native-import": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "log", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers-out", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--eval-out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi eval native-import", - "subcommand_required": null, - "subcommands": {} - }, - "pairwise": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--arms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--baseline-arm", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--escalate-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--escalate-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--escalate-ties", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-temperature", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi eval pairwise", - "subcommand_required": null, - "subcommands": {} - }, - "panel-score": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 6, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-configs", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-temperature", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--limit", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--roles", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi eval panel-score", - "subcommand_required": null, - "subcommands": {} - }, - "ragas": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--ragas-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'anthropic'", - "'openai'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--ragas-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi eval ragas", - "subcommand_required": null, - "subcommands": {} - }, - "redteam": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "questions", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.redteam._judge_ref#a59ddb60ae31" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi eval redteam", - "subcommand_required": null, - "subcommands": {} - }, - "reference": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 4, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-judge", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output/-o", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--questions", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - } - ], - "path": "dokimi eval reference", - "subcommand_required": null, - "subcommands": {} - }, - "score": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--answers", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--judge-placeholder", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-temperature", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--run-id", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi eval score", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "evaluate": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--context7-id", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--custom-questions", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--debug-retrieval", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--description", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "context7", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--doc-source", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--judge-placeholder", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-temperature", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": ".", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--personas-count", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 2, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--questions-per-topic", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--questions-root", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--repo", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0.3, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--rerank-threshold", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--top-k", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - } - ], - "path": "dokimi evaluate", - "subcommand_required": null, - "subcommands": {} - }, - "explore": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi explore", - "subcommand_required": true, - "subcommands": { - "list": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--repo-root", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--tasks-root", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi explore list", - "subcommand_required": null, - "subcommands": {} - }, - "run": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--all", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "oracle,empty", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--arms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--baseline-arm", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out-json", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out-md", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "results/explore-runs", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--repo-root", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--tasks", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--tasks-root", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi explore run", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "ingest": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi ingest", - "subcommand_required": true, - "subcommands": { - "docling": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--input", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-recursive", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - } - ], - "path": "dokimi ingest docling", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "library": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi library", - "subcommand_required": true, - "subcommands": { - "intents": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--intents", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi library intents", - "subcommand_required": null, - "subcommands": {} - }, - "list": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--domain", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--intents", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--registry", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi library list", - "subcommand_required": null, - "subcommands": {} - }, - "show": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "name", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--intents", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--registry", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi library show", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "novelty": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi novelty", - "subcommand_required": true, - "subcommands": { - "analyze": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--all-atoms", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--allow-self-judge", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "no_artifact,full", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--arms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--artifact", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": [ - 0.75, - 0.5 - ], - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--budgets", - "kind": "option", - "nargs": "+", - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--estimate-only", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--extract-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--extract-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--include-resources", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-atoms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 4000, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-file-words", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 20000, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-resource-words", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-tokens", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-usd", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.novelty._usd#591d6e2f0474" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-classify", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-confirm-carriers", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-paraphrase", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-reversal", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-shorten", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-verify", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out-json", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--placebo-text", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'data'", - "'eval'", - "'other'", - "'pattern'", - "'reference'", - "'script'", - "'template'", - "'top'", - "'trigger'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--resource-roles", - "kind": "option", - "nargs": "+", - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--seed", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreAction", - "choices": [ - "'standard'", - "'study'", - "'triage'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--tier", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'html'", - "'md'" - ], - "const": null, - "default": "html", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--view-format", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi novelty analyze", - "subcommand_required": null, - "subcommands": {} - }, - "atoms": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--all-atoms", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--artifact", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": [ - 0.75, - 0.5 - ], - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--budgets", - "kind": "option", - "nargs": "+", - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--extract-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--include-resources", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-atoms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 4000, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-file-words", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 20000, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-resource-words", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-tokens", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-usd", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.novelty._usd#591d6e2f0474" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-verify", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'data'", - "'eval'", - "'other'", - "'pattern'", - "'reference'", - "'script'", - "'template'", - "'top'", - "'trigger'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--resource-roles", - "kind": "option", - "nargs": "+", - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--seed", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - } - ], - "path": "dokimi novelty atoms", - "subcommand_required": null, - "subcommands": {} - }, - "items": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "full", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--arm", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--probes", - "kind": "option", - "nargs": "+", - "position": null, - "required": true, - "type": null - } - ], - "path": "dokimi novelty items", - "subcommand_required": null, - "subcommands": {} - }, - "probe": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--all-atoms", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--allow-self-judge", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "no_artifact,full", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--arms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--artifact", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--atoms", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": [ - 0.75, - 0.5 - ], - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--budgets", - "kind": "option", - "nargs": "+", - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--extract-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--extract-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-atoms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-tokens", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-usd", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.novelty._usd#591d6e2f0474" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-paraphrase", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--no-reversal", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--placebo-text", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--seed", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - } - ], - "path": "dokimi novelty probe", - "subcommand_required": null, - "subcommands": {} - }, - "routing": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--all-vs-all", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--artifact", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--catalog", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--competitors", - "kind": "option", - "nargs": "+", - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--include-disabled", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-tokens", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-usd", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.novelty._usd#591d6e2f0474" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--queries", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--queries-by-skill", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--queries-from-headings", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 1, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--repeats", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--seed", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - } - ], - "path": "dokimi novelty routing", - "subcommand_required": null, - "subcommands": {} - }, - "shorten": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--all-atoms", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--allow-unsafe", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--artifact", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--atoms", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": [ - 0.75, - 0.5 - ], - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--budgets", - "kind": "option", - "nargs": "+", - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--extract-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--lexical-only", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-atoms", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-tokens", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-usd", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.novelty._usd#591d6e2f0474" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--seed", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - } - ], - "path": "dokimi novelty shorten", - "subcommand_required": null, - "subcommands": {} - }, - "view": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--atoms", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'html'", - "'md'" - ], - "const": null, - "default": "html", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--format", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--novelty", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi novelty view", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "personas": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi personas", - "subcommand_required": true, - "subcommands": { - "approve": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--file", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi personas approve", - "subcommand_required": null, - "subcommands": {} - }, - "discover": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--count", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--description", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--github-token", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--repo", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--save-analysis", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi personas discover", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "product": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi product", - "subcommand_required": true, - "subcommands": { - "intents": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--intents", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi product intents", - "subcommand_required": null, - "subcommands": {} - }, - "list": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--domain", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--intents", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--registry", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi product list", - "subcommand_required": null, - "subcommands": {} - }, - "show": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "name", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--intents", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--registry", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi product show", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "questions": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi questions", - "subcommand_required": true, - "subcommands": { - "analyze": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 5, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--questions", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - } - ], - "path": "dokimi questions analyze", - "subcommand_required": null, - "subcommands": {} - }, - "generate": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--context7-id", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 2, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--count", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "context7", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--doc-source", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'auto'", - "'local'", - "'openai'" - ], - "const": null, - "default": "auto", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--embeddings", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--personas", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--top-k", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--topics", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--validate", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi questions generate", - "subcommand_required": null, - "subcommands": {} - }, - "import": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "path", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--format", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--grader", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output/-o", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--scorer", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--task", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi questions import", - "subcommand_required": null, - "subcommands": {} - }, - "panel-review": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 6, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--concurrency", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--limit", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'claude-cli'", - "'google-vertex'", - "'openai'", - "'openai-codex'", - "'openrouter'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--questions", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--reviewers", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi questions panel-review", - "subcommand_required": null, - "subcommands": {} - }, - "refine": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--dry-run", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--fill-gaps", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "gpt-6.1-sol", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--fill-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'anthropic'", - "'google'", - "'openai'" - ], - "const": null, - "default": "openai", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--fill-provider", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--output", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--product", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--questions", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0.82, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--sim-threshold", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 10, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--target-advanced", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 10, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--target-beginner", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 10, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--target-intermediate", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "int" - } - ], - "path": "dokimi questions refine", - "subcommand_required": null, - "subcommands": {} - } - } - }, - "repo": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo", - "subcommand_required": true, - "subcommands": { - "budget": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "source", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--fail-on-warning", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-code", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--harness", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--json", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo budget", - "subcommand_required": null, - "subcommands": {} - }, - "compare": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "batch_dirs", - "kind": "argument", - "nargs": "+", - "position": 0, - "required": true, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--arm", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--baseline", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--json", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--metric", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--pass-k", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--path-stability", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--task", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo compare", - "subcommand_required": null, - "subcommands": {} - }, - "fetch": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "source", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--json", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo fetch", - "subcommand_required": null, - "subcommands": {} - }, - "inspect": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "source", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--fail-on-error", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--json", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 100000, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--large-bytes", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo inspect", - "subcommand_required": null, - "subcommands": {} - }, - "reach": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "source", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--bin", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 0.1, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--budget", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "float" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--dry-run", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-code", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--harness", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--keep", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--manifest", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--manifest-file", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 4, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-turns", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--model", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--only", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "reach-out", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--salt", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "root,cwd,read", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--scenarios", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 300, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--timeout", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--work-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo reach", - "subcommand_required": null, - "subcommands": {} - }, - "report": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "batch_dir", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--hits", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--json", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo report", - "subcommand_required": null, - "subcommands": {} - }, - "run": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "source", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null + "path": "dokimi eval judge", + "subcommand_required": true, + "subcommands": { + "agreement": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--baseline-arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--both", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--checklist", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--grounded", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judges", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-stored", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--pairwise", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--questions", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--require-validated", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--strict", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--validity", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval judge agreement", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreAction", - "choices": [ - "'bridge'", - "'host'" + "validate": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 20, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--bases", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": [ + "'none'", + "'stored'" + ], + "const": null, + "default": "none", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--context", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--escalate-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--escalate-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--escalate-ties", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 120, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-bases", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.8, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-accuracy", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": [ + "'absolute'", + "'pairwise'" + ], + "const": null, + "default": "absolute", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--mode", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "judge", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--scorer", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--seed", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--sequential", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--strict", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--typed-max-len", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.evaluate._non_negative_int#95d3497f2297" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--typed-threads", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--variants", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } ], - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--agent-network", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--arm", - "kind": "option", - "nargs": null, - "position": null, - "required": true, - "type": null + "path": "dokimi eval judge validate", + "subcommand_required": null, + "subcommands": {} }, + "wording": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 20, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--bases", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.8, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-accuracy", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 1, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--redraws", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--seed", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--strict", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--variants", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--wordings", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval judge wording", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "native-import": { + "allow_abbrev": true, + "params": [ { "action": "_StoreAction", "choices": null, @@ -11323,10 +5264,10 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--batch", - "kind": "option", + "key": "log", + "kind": "argument", "nargs": null, - "position": null, + "position": 0, "required": true, "type": null }, @@ -11339,29 +5280,13 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--bin", + "key": "--answers-out", "kind": "option", "nargs": null, "position": null, - "required": false, + "required": true, "type": null }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 2.0, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--budget", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.repo_run._usd#591d6e2f0474" - }, { "action": "_StoreAction", "choices": null, @@ -11371,7 +5296,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--build-network", + "key": "--eval-out", "kind": "option", "nargs": null, "position": null, @@ -11379,306 +5304,1041 @@ "type": null }, { - "action": "_StoreAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": 10800, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--build-timeout", + "is_flag": true, + "key": "--help/-h", "kind": "option", - "nargs": null, + "nargs": 0, "position": null, "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, + "type": null + } + ], + "path": "dokimi eval native-import", + "subcommand_required": null, + "subcommands": {} + }, + "personas": { + "allow_abbrev": true, + "params": [ { - "action": "_AppendAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": null, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--cache", + "is_flag": true, + "key": "--help/-h", "kind": "option", - "nargs": null, + "nargs": 0, "position": null, "required": false, "type": null + } + ], + "path": "dokimi eval personas", + "subcommand_required": true, + "subcommands": { + "discover": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--count", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--description", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--github-token", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--repo", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--save-analysis", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval personas discover", + "subcommand_required": null, + "subcommands": {} }, + "validate": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--file", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval personas validate", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "questions": { + "allow_abbrev": true, + "params": [ { - "action": "_StoreAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": null, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--cache-dir", + "is_flag": true, + "key": "--help/-h", "kind": "option", - "nargs": null, + "nargs": 0, "position": null, "required": false, "type": null + } + ], + "path": "dokimi eval questions", + "subcommand_required": true, + "subcommands": { + "analyze": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--questions", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + } + ], + "path": "dokimi eval questions analyze", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-root", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null + "generate": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--context7-id", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 2, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--count", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "context7", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--doc-source", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'auto'", + "'local'", + "'openai'" + ], + "const": null, + "default": "auto", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--embeddings", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--personas", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--top-k", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--topics", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--validate", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval questions generate", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreAction", - "choices": [ - "'native'", - "'none'" + "refine": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--dry-run", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--fill-gaps", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--fill-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'anthropic'", + "'google'", + "'openai'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--fill-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--questions", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.82, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--sim-threshold", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 10, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--target-advanced", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 10, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--target-beginner", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 10, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--target-intermediate", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + } ], - "const": null, - "default": "native", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--delegation", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "docker", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--docker", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--force", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 3600, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--grade-timeout", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": "claude-code", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--harness", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--image", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 1, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--jobs/-j", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": ".agent-eval/repo-eval.yaml", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--manifest", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--manifest-file", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-batch-usd", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.commands.repo_run._usd#591d6e2f0474" + "path": "dokimi eval questions refine", + "subcommand_required": null, + "subcommands": {} }, + "review": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 6, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--limit", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--questions", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--reviewers", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval questions review", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "redteam": { + "allow_abbrev": true, + "params": [ { "action": "_StoreAction", "choices": null, "const": null, - "default": 200, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--max-turns", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--model", - "kind": "option", + "key": "questions", + "kind": "argument", "nargs": null, - "position": null, + "position": 0, "required": true, "type": null }, { - "action": "_StoreAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": "repo-runs", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, - "key": "--rebuild-image", + "key": "--help/-h", "kind": "option", "nargs": 0, "position": null, @@ -11689,76 +6349,28 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 1, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--reps", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--source-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_AppendAction", - "choices": null, - "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--task", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": 2700, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--timeout", + "key": "--judge", "kind": "option", "nargs": null, "position": null, "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + "type": "dokimi.commands.redteam._judge_ref#a59ddb60ae31" }, { "action": "_StoreAction", "choices": null, "const": null, - "default": "all", + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--tools", + "key": "--out", "kind": "option", "nargs": null, "position": null, @@ -11766,61 +6378,13 @@ "type": null } ], - "path": "dokimi repo run", + "path": "dokimi eval redteam", "subcommand_required": null, "subcommands": {} }, - "scan": { + "score": { "allow_abbrev": true, "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "source", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--cache-dir", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--fail-on-fail", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, { "action": "_HelpAction", "choices": null, @@ -11836,116 +6400,860 @@ "position": null, "required": false, "type": null + } + ], + "path": "dokimi eval score", + "subcommand_required": true, + "subcommands": { + "checklist": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--baseline-arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--questions", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + } + ], + "path": "dokimi eval score checklist", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--json", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null + "grounding": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--threshold", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + } + ], + "path": "dokimi eval score grounding", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--manifest", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null + "pairwise": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--baseline-arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--escalate-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--escalate-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--escalate-ties", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval score pairwise", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--manifest-file", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null + "panel": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 6, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-configs", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--limit", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--roles", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval score panel", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null + "pointwise": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--answers", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--run-id", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval score pointwise", + "subcommand_required": null, + "subcommands": {} }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--static", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo scan", - "subcommand_required": null, - "subcommands": {} - }, - "tasks": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi repo tasks", - "subcommand_required": true, - "subcommands": { - "check": { + "ragas": { "allow_abbrev": true, "params": [ { @@ -11957,23 +7265,23 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "source", - "kind": "argument", + "key": "--answers", + "kind": "option", "nargs": null, - "position": 0, + "position": null, "required": true, "type": null }, { - "action": "_StoreTrueAction", + "action": "_HelpAction", "choices": null, - "const": true, - "default": false, + "const": null, + "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, - "key": "--allow-unsupported", + "key": "--help/-h", "kind": "option", "nargs": 0, "position": null, @@ -11981,21 +7289,48 @@ "type": null }, { - "action": "_AppendAction", + "action": "_StoreAction", "choices": null, "const": null, - "default": null, + "default": "claude-sonnet-5-5", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--arm", + "key": "--ragas-model", "kind": "option", "nargs": null, "position": null, "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": [ + "'anthropic'", + "'openai'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--ragas-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval score ragas", + "subcommand_required": null, + "subcommands": {} + }, + "reference": { + "allow_abbrev": true, + "params": [ { "action": "_StoreAction", "choices": null, @@ -12005,23 +7340,23 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--build-network", + "key": "--answers", "kind": "option", "nargs": null, "position": null, - "required": false, + "required": true, "type": null }, { "action": "_StoreAction", "choices": null, "const": null, - "default": 10800, + "default": 4, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--build-timeout", + "key": "--concurrency", "kind": "option", "nargs": null, "position": null, @@ -12029,21 +7364,77 @@ "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" }, { - "action": "_AppendAction", + "action": "_HelpAction", "choices": null, "const": null, - "default": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--cache", + "key": "--judge-provider", "kind": "option", "nargs": null, "position": null, "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-judge", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -12053,11 +7444,11 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--cache-dir", + "key": "--output/-o", "kind": "option", "nargs": null, "position": null, - "required": false, + "required": true, "type": null }, { @@ -12069,10 +7460,59 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--cache-root", + "key": "--questions", "kind": "option", "nargs": null, "position": null, + "required": true, + "type": null + } + ], + "path": "dokimi eval score reference", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "tasks": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval tasks", + "subcommand_required": true, + "subcommands": { + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, "required": false, "type": null }, @@ -12080,33 +7520,86 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": "docker", + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--docker", + "key": "--tasks-root", "kind": "option", "nargs": null, "position": null, "required": false, "type": null - }, + } + ], + "path": "dokimi eval tasks list", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "triggers": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval triggers", + "subcommand_required": true, + "subcommands": { + "generate": { + "allow_abbrev": true, + "params": [ { "action": "_StoreAction", "choices": null, "const": null, - "default": 3600, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--catalog", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'questions'", + "'reach'", + "'skill-creator'" + ], + "const": null, + "default": "reach", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--grade-timeout", + "key": "--format", "kind": "option", "nargs": null, "position": null, "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + "type": null }, { "action": "_HelpAction", @@ -12133,7 +7626,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--image", + "key": "--model", "kind": "option", "nargs": null, "position": null, @@ -12144,28 +7637,28 @@ "action": "_StoreAction", "choices": null, "const": null, - "default": 1, + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--jobs/-j", + "key": "--out", "kind": "option", "nargs": null, "position": null, - "required": false, - "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + "required": true, + "type": null }, { "action": "_StoreAction", "choices": null, "const": null, - "default": ".agent-eval/repo-eval.yaml", + "default": "openai", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--manifest", + "key": "--provider", "kind": "option", "nargs": null, "position": null, @@ -12173,7 +7666,7 @@ "type": null }, { - "action": "_StoreAction", + "action": "_AppendAction", "choices": null, "const": null, "default": null, @@ -12181,7 +7674,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--manifest-file", + "key": "--questions", "kind": "option", "nargs": null, "position": null, @@ -12189,17 +7682,17 @@ "type": null }, { - "action": "_StoreAction", + "action": "_StoreTrueAction", "choices": null, - "const": null, - "default": "repo-check", + "const": true, + "default": false, "group_key": null, "group_required": null, "hidden": false, - "is_flag": false, - "key": "--out", + "is_flag": true, + "key": "--rewrite", "kind": "option", - "nargs": null, + "nargs": 0, "position": null, "required": false, "type": null @@ -12213,7 +7706,7 @@ "group_required": null, "hidden": false, "is_flag": true, - "key": "--rebuild-image", + "key": "--strict", "kind": "option", "nargs": 0, "position": null, @@ -12229,7 +7722,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--source-dir", + "key": "--target", "kind": "option", "nargs": null, "position": null, @@ -12245,34 +7738,4753 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--task", + "key": "--trigger-set", "kind": "option", "nargs": null, "position": null, "required": false, "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--warm", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null } ], - "path": "dokimi repo tasks check", - "subcommand_required": null, - "subcommands": {} + "path": "dokimi eval triggers generate", + "subcommand_required": null, + "subcommands": {} + } + } + } + } + }, + "evaluate": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--context7-id", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--custom-questions", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--debug-retrieval", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--description", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "context7", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--doc-source", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": ".", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--personas-count", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--product", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 2, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--questions-per-topic", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--questions-root", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--repo", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.3, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--rerank-threshold", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--top-k", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + } + ], + "path": "dokimi evaluate", + "subcommand_required": null, + "subcommands": {} + }, + "explore": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi explore", + "subcommand_required": true, + "subcommands": { + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--repo-root", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--tasks-root", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi explore list", + "subcommand_required": null, + "subcommands": {} + }, + "run": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--all", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "oracle,empty", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--baseline-arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out-json", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out-md", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "results/explore-runs", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--repo-root", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--tasks", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--tasks-root", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi explore run", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "ingest": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi ingest", + "subcommand_required": true, + "subcommands": { + "docling": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--input", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-recursive", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + } + ], + "path": "dokimi ingest docling", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "library": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi library", + "subcommand_required": true, + "subcommands": { + "intents": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--intents", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi library intents", + "subcommand_required": null, + "subcommands": {} + }, + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--domain", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--intents", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--registry", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi library list", + "subcommand_required": null, + "subcommands": {} + }, + "show": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "name", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--intents", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--registry", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi library show", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "novelty": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi novelty", + "subcommand_required": true, + "subcommands": { + "analyze": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--all-atoms", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--allow-self-judge", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "no_artifact,full", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--artifact", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": [ + 0.75, + 0.5 + ], + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--budgets", + "kind": "option", + "nargs": "+", + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--estimate-only", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--extract-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--extract-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--include-resources", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-atoms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 4000, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-file-words", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 20000, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-resource-words", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-tokens", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-usd", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.novelty._usd#591d6e2f0474" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-classify", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-confirm-carriers", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-paraphrase", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-reversal", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-shorten", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-verify", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out-json", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--placebo-text", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'data'", + "'eval'", + "'other'", + "'pattern'", + "'reference'", + "'script'", + "'template'", + "'top'", + "'trigger'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--resource-roles", + "kind": "option", + "nargs": "+", + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--seed", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + }, + { + "action": "_StoreAction", + "choices": [ + "'standard'", + "'study'", + "'triage'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--tier", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'html'", + "'md'" + ], + "const": null, + "default": "html", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--view-format", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi novelty analyze", + "subcommand_required": null, + "subcommands": {} + }, + "atoms": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--all-atoms", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--artifact", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": [ + 0.75, + 0.5 + ], + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--budgets", + "kind": "option", + "nargs": "+", + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--extract-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--include-resources", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-atoms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 4000, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-file-words", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 20000, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-resource-words", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-tokens", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-usd", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.novelty._usd#591d6e2f0474" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-verify", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'data'", + "'eval'", + "'other'", + "'pattern'", + "'reference'", + "'script'", + "'template'", + "'top'", + "'trigger'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--resource-roles", + "kind": "option", + "nargs": "+", + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--seed", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + } + ], + "path": "dokimi novelty atoms", + "subcommand_required": null, + "subcommands": {} + }, + "items": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "full", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--probes", + "kind": "option", + "nargs": "+", + "position": null, + "required": true, + "type": null + } + ], + "path": "dokimi novelty items", + "subcommand_required": null, + "subcommands": {} + }, + "probe": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--all-atoms", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--allow-self-judge", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "no_artifact,full", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--artifact", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--atoms", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": [ + 0.75, + 0.5 + ], + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--budgets", + "kind": "option", + "nargs": "+", + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 5, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--concurrency", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--extract-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--extract-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-atoms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-tokens", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-usd", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.novelty._usd#591d6e2f0474" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "gpt-6.1-sol", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-paraphrase", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-reversal", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--placebo-text", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--seed", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + } + ], + "path": "dokimi novelty probe", + "subcommand_required": null, + "subcommands": {} + }, + "routing": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--all-vs-all", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--artifact", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--catalog", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--competitors", + "kind": "option", + "nargs": "+", + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--include-disabled", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-tokens", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-usd", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.novelty._usd#591d6e2f0474" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "openai", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--queries", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--queries-by-skill", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--queries-from-headings", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 1, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--repeats", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--seed", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + } + ], + "path": "dokimi novelty routing", + "subcommand_required": null, + "subcommands": {} + }, + "shorten": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--all-atoms", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--allow-unsafe", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--artifact", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--atoms", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": [ + 0.75, + 0.5 + ], + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--budgets", + "kind": "option", + "nargs": "+", + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--extract-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--lexical-only", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-atoms", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-tokens", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-usd", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.novelty._usd#591d6e2f0474" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'claude-cli'", + "'google-vertex'", + "'openai'", + "'openai-codex'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--seed", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "int" + } + ], + "path": "dokimi novelty shorten", + "subcommand_required": null, + "subcommands": {} + }, + "view": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--atoms", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'html'", + "'md'" + ], + "const": null, + "default": "html", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--format", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--novelty", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi novelty view", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "product": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi product", + "subcommand_required": true, + "subcommands": { + "intents": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--intents", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi product intents", + "subcommand_required": null, + "subcommands": {} + }, + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--domain", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--intents", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--registry", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi product list", + "subcommand_required": null, + "subcommands": {} + }, + "show": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "name", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--intents", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--registry", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi product show", + "subcommand_required": null, + "subcommands": {} + } + } + }, + "repo": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo", + "subcommand_required": true, + "subcommands": { + "budget": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "source", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--fail-on-warning", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-code", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--harness", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo budget", + "subcommand_required": null, + "subcommands": {} + }, + "compare": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "batch_dirs", + "kind": "argument", + "nargs": "+", + "position": 0, + "required": true, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arm", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--baseline", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--metric", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--pass-k", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--path-stability", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--task", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo compare", + "subcommand_required": null, + "subcommands": {} + }, + "fetch": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "source", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo fetch", + "subcommand_required": null, + "subcommands": {} + }, + "inspect": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "source", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--fail-on-error", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 100000, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--large-bytes", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo inspect", + "subcommand_required": null, + "subcommands": {} + }, + "reach": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "source", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--bin", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 0.1, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--budget", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--dry-run", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-code", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--harness", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--keep", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--manifest", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--manifest-file", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 4, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-turns", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--only", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "reach-out", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--salt", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "root,cwd,read", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--scenarios", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 300, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--timeout", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--work-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo reach", + "subcommand_required": null, + "subcommands": {} + }, + "report": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "batch_dir", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--hits", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo report", + "subcommand_required": null, + "subcommands": {} + }, + "run": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "source", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'bridge'", + "'host'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--agent-network", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--arm", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--batch", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--bin", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 2.0, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--budget", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.repo_run._usd#591d6e2f0474" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--build-network", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 10800, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--build-timeout", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-root", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'native'", + "'none'" + ], + "const": null, + "default": "native", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--delegation", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "docker", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--docker", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--force", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 3600, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--grade-timeout", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-code", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--harness", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--image", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 1, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--jobs/-j", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": ".agent-eval/repo-eval.yaml", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--manifest", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--manifest-file", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-batch-usd", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.repo_run._usd#591d6e2f0474" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 200, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-turns", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--model", + "kind": "option", + "nargs": null, + "position": null, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "repo-runs", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--rebuild-image", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 1, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--reps", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--source-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--task", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": 2700, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--timeout", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "all", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--tools", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo run", + "subcommand_required": null, + "subcommands": {} + }, + "scan": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "source", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--cache-dir", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--fail-on-fail", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--manifest", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--manifest-file", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--static", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo scan", + "subcommand_required": null, + "subcommands": {} + }, + "tasks": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi repo tasks", + "subcommand_required": true, + "subcommands": { "replay": { "allow_abbrev": true, "params": [ @@ -13917,118 +14129,6 @@ "subcommand_required": null, "subcommands": {} }, - "inspect": { - "allow_abbrev": true, - "params": [ - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "path", - "kind": "argument", - "nargs": null, - "position": 0, - "required": true, - "type": null - }, - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--json", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--out", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi tasks inspect", - "subcommand_required": null, - "subcommands": {} - }, - "list": { - "allow_abbrev": true, - "params": [ - { - "action": "_HelpAction", - "choices": null, - "const": null, - "default": "==SUPPRESS==", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--help/-h", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": null, - "const": null, - "default": null, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--tasks-root", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - } - ], - "path": "dokimi tasks list", - "subcommand_required": null, - "subcommands": {} - }, "run": { "allow_abbrev": true, "params": [ diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index a26a0c92c..3895f3214 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -77,7 +77,7 @@ SPDX-License-Identifier: Apache-2.0 > and agent runs with their report: > > ```bash -> python cli.py repo tasks check --manifest [--task ] [--warm] [--allow-unsupported] +> python cli.py eval check --manifest [--task ] [--warm] [--allow-unsupported] > python cli.py repo run --manifest --arm guidance:raw --model --reps N --batch NAME > --agent-network host > python cli.py repo report / @@ -160,7 +160,7 @@ outside every workspace and passes its absolute path to entry points as as `.bazelversion`) is read at run time, not baked in. A build that runs past `--build-timeout` seconds (3 hours by default) fails as a container error rather than hanging the batch. -- `tasks check` (§4.3) fails a task whose agent container can read any file +- `eval check` (§4.3) fails a task whose agent container can read any file under `tasks` or `{source_dir}`. ### 1.2 Code index input @@ -233,7 +233,7 @@ Rules: - Every path is relative to the repository root. - `eval_dir`, `tasks` and `entry_points` come together or not at all. Without them the manifest is *guidance-only*: `scan` and `reach` run on it and `validate` checks its - globs, while `tasks check` and `run` refuse it. `image` and `explore_tasks` need + globs, while `eval check` and `run` refuse it. `image` and `explore_tasks` need `eval_dir`, and so does `process_metrics`. - `entry_points.grade` is required only when some task's `grader` does not start with `dokimi:` (a grader built into the framework); `repo validate` @@ -269,7 +269,7 @@ Rules: dependency into the image is better when it can be done. - **Caches** are mounted read-write in every run, because Bazel's disk cache writes. A run may depend on them for speed, never for correctness. With - `network: none` they are filled by `tasks check --warm` (§4.3): one base is + `network: none` they are filled by `eval check --warm` (§4.3): one base is not enough when bases pin different tool versions (7 Bazel versions on oneDAL). - **Container runtime.** The framework starts every container with an init @@ -543,11 +543,11 @@ measure the patch's text, such as a guidance line, not the tool. [repo-evaluation-mergeability.md](repo-evaluation-mergeability.md) *(proposed)*. The judge is not deterministic. Instead of a determinism check, it reports agreement between two judges and must pass a calibration - gate (`repo rubric check`, a separate paid subcommand; `tasks check` stays + gate (`repo rubric check`, a separate paid subcommand; `eval check` stays free of LLM calls). The regrade sample bypasses the verdict cache. Whether one judge repeats its own verdict on the same input is not measured yet. -### 4.3 Self-check (`tasks check`, no LLM) +### 4.3 Self-check (`eval check`, no LLM) For every task, before any paid run: @@ -559,7 +559,7 @@ For every task, before any paid run: A task whose `host` requirements (§3) the probe container does not meet is `unsupported` with the reason, like a task whose `egress` needs the missing -proxy, and `tasks check` exits 1 for it. With `--allow-unsupported` it is +proxy, and `eval check` exits 1 for it. With `--allow-unsupported` it is still printed with its reason but does not make the exit code 1 unless no task is ok (below); broken tasks, tasks that need egress and tasks whose host probe failed still do. The probed facts are recorded in `check.json` `host`, and for @@ -570,14 +570,14 @@ apart, neither failed nor missing. If the probe itself fails, a task with cells is left unrun; a cell that already has a `result.json` keeps it. Nothing checked is not a pass. Both commands refuse an export with no task, and -exit 1 as `incomplete` when nothing was measured: `tasks check` when no task +exit 1 as `incomplete` when nothing was measured: `eval check` when no task passed the self-check (every one unsupported, even with `--allow-unsupported`), `repo run` when no cell of the batch has a result (every one unsupported). The `repo run` summary line counts unsupported cells beside ran, done, crashed and not started. Each task runs with its own network (`image.network` plus its `egress`). -`tasks check --warm` runs the same null and oracle steps with network and +`eval check --warm` runs the same null and oracle steps with network and writable caches, to fill the caches before offline runs (§2); it judges task `host` requirements as an offline check does. @@ -602,7 +602,7 @@ A task whose `grader` is `dokimi:` is graded by the framework, not by the repository's `grade` and `oracle` entry points; any other name goes to those entry points. Such a task is data only: its directory holds `task.json`, the patch and the JSON files the grader reads. The `dokimi:` -prefix is the framework's: an unknown `dokimi:` name fails `tasks check` and +prefix is the framework's: an unknown `dokimi:` name fails `eval check` and the run's grade, and is never handed to the repository. A built-in grader runs where `grade` and `oracle` run: in the grading @@ -820,7 +820,7 @@ on an error that is not one pull request's own: a malformed pull request, the source, the manifest, the prompt file, or a task it cannot write. Each prints `✗` and the reason. A git failure while one pull request is built skips that pull request with the reason. -`repo tasks check` then runs the tasks, and two arms measure the guidance +`eval check` then runs the tasks, and two arms measure the guidance (§9): ```bash @@ -1365,7 +1365,7 @@ finding is an error. ## 9. Running -`repo tasks check` and `repo run` need docker. Both export `eval_dir` from the +`eval check` and `repo run` need docker. Both export `eval_dir` from the eval tree to `/_eval/content`, and build the image from the eval tree without its tasks directory (tag `dokimi/:`) unless `--image` names one. Cache directories default to @@ -1391,7 +1391,7 @@ line, else its last `error:` line, else its last line. ```bash # self-check: no LLM; exits 1 if any task is broken or unsupported, or none is ok # (with --allow-unsupported, a task this host cannot run does not fail it on its own) -python cli.py repo tasks check ~/oneDAL@ --manifest dev/agent_evals/repo-eval.yaml \ +python cli.py eval check ~/oneDAL@ --manifest dev/agent_evals/repo-eval.yaml \ --task fix_finiteness_mut --build-network host --cache bazel-repo-cache=/cache/bazel-repo -j 3 # agents: one cell per (task, arm, model, rep) under repo-runs// @@ -1582,7 +1582,7 @@ that runs the no-LLM checks on the caller's checkout and needs no secret. - `validate` runs `repo validate` and gates the other two jobs; `scan` runs `repo scan` (report only: the claim counts and failing claims go to the job summary, the - `repo_readiness.v1` report to an artifact); `tasks-check` runs `repo tasks check` + `repo_readiness.v1` report to an artifact); `tasks-check` runs `eval check` for each selected task in a job of its own (docker; the image is built from the manifest's Dockerfile), and uploads `check.json` and the grade logs. If the job is cancelled by its `timeout-minutes` before the check finishes, the Summary step diff --git a/dokimi/docs/repo-evaluation-mergeability.md b/dokimi/docs/repo-evaluation-mergeability.md index e5cf3051f..348fbf311 100644 --- a/dokimi/docs/repo-evaluation-mergeability.md +++ b/dokimi/docs/repo-evaluation-mergeability.md @@ -225,7 +225,7 @@ tests: ## 5. Admitting a task -`tasks check` (contract §4.3) stays free of LLM calls. The paid step is a +`eval check` (contract §4.3) stays free of LLM calls. The paid step is a separate subcommand, `repo rubric check`, which runs once per rubric revision, not once per sweep. A task is admitted only if: diff --git a/dokimi/docs/reviewing.md b/dokimi/docs/reviewing.md index 6fa1a3ff1..414e3c12a 100644 --- a/dokimi/docs/reviewing.md +++ b/dokimi/docs/reviewing.md @@ -33,7 +33,7 @@ credentials (`GOOGLE_APPLICATION_CREDENTIALS`, with `VERTEXAI_PROJECT` and gateway), use provider `openai` with `OPENAI_API_BASE` (or `OPENAI_BASE_URL`) set to its `/v1` URL. A model name that litellm does not take for OpenAI's own goes to that server, such as `glm-5.3` or `meta-llama/Llama-3.1-8B` -(`--judges openai/glm-5.3` in `eval cross-judge`); so does a bare `claude-*`, +(`--judges openai/glm-5.3` in `eval judge agreement`); so does a bare `claude-*`, Bedrock or Vertex id, which needs its cloud's provider to reach that cloud. A name that starts with a litellm provider prefix (`deepseek/`, `groq/`, `ollama/`) keeps that provider's route: write it as `openai/` to send it diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md index 140f941dd..64647a764 100644 --- a/dokimi/docs/runs.md +++ b/dokimi/docs/runs.md @@ -73,9 +73,9 @@ into runs and `run_id` is `null`. Files outside the root are not listed: - `repo run` writes its batches to `repo-runs/`, and its files carry no `schema_version`; -- `answers generate` and `eval score` write to `answers/` and `eval/` unless - `--run-id` is given, and `eval panel-score` writes to `eval/`; -- `questions generate` and `questions refine` write to `questions/`, +- `answers generate` and `eval score pointwise` write to `answers/` and `eval/` unless + `--run-id` is given, and `eval score panel` writes to `eval/`; +- `eval questions generate` and `eval questions refine` write to `questions/`, `repo reach` to `reach-out/`, `tools preference` to `tool_preference.json` in the working directory; - an output flag can point anywhere. diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 23a2cd8e2..c90e907db 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -35,7 +35,7 @@ exits 2 for a backend that is not installed or not signed in, as for a missing k ```bash python cli.py answers generate --product oneTBB --questions questions/oneTBB.json \ --output answers/oneTBB.json --provider claude-cli --model haiku -python cli.py eval score --product oneTBB --answers answers/oneTBB.json \ +python cli.py eval score pointwise --product oneTBB --answers answers/oneTBB.json \ --output eval/oneTBB.json --judge-provider claude-cli --judge-model sonnet ``` diff --git a/dokimi/docs/user-journeys/flow-map.md b/dokimi/docs/user-journeys/flow-map.md index 6532dd36b..08c625eb5 100644 --- a/dokimi/docs/user-journeys/flow-map.md +++ b/dokimi/docs/user-journeys/flow-map.md @@ -21,7 +21,7 @@ flowchart LR J1 -. proposed: costed next-step plan .-> plan[Next evaluation plan] J1 --> J4[J4 guidance: correctness, reach] J1 --> J2[J2 diagnose skill: routing, novelty] - J1 --> J3[J3 repository: scan, tasks check, runs] + J1 --> J3[J3 repository: scan, eval check, runs] J2 --> change[Change the subject] J4 --> change J3 --> change diff --git a/dokimi/docs/user-journeys/journeys.md b/dokimi/docs/user-journeys/journeys.md index c49317040..39a1ac774 100644 --- a/dokimi/docs/user-journeys/journeys.md +++ b/dokimi/docs/user-journeys/journeys.md @@ -201,7 +201,7 @@ whether Docker is available; the budget for agent runs. tree (J7). 5. Self-check the tasks before any agent run: the untouched workspace must fail, the reference solution must pass, the agent must not see the graders - (`python cli.py repo tasks check`). + (`python cli.py eval check`). 6. Run agents on the tasks with and without the guidance (`python cli.py repo run`), resuming without re-running finished cells. 7. Report one batch and compare arms (`python cli.py repo report`, @@ -437,9 +437,9 @@ Questions: what users do with the subject; which failures matter most. 1. Validate what exists (`python cli.py repo validate --manifest`). 2. Self-check graders with no model: null, oracle, isolation - (`python cli.py repo tasks check`). + (`python cli.py eval check`). 3. For judged evals, check the judge: does it tell good answers from - degraded ones (`python cli.py eval judge-validity`)? + degraded ones (`python cli.py eval judge validate`)? 4. Propose new tasks from the subject's signals, as hypotheses for the user to confirm. **Proposed.** 5. Add negative controls: tasks the subject should not help with, or @@ -719,7 +719,7 @@ cost. non-zero exit only when a model cannot be verified or differs from the baseline; a regression in the scores is printed and exits 0. Failing on a regression beyond a threshold is **proposed** (AC-J12-001). Blocking policy is **partial** -(AC-J12-002): `repo tasks check --allow-unsupported` decides host-unsupported +(AC-J12-002): `eval check --allow-unsupported` decides host-unsupported tasks; budget-stopped and inconclusive judge results have no policy. ### Use cases and acceptance @@ -792,7 +792,7 @@ the settings that produced it, the spend, and the stated limits. | Skill routing | `python cli.py novelty routing` | Free without `--model` | | Guidance reach | `python cli.py repo reach` | Plan with `--dry-run`; cap with `--budget` | | Context window of a run | `python cli.py context trace` | Reads an existing trace | -| Judge calibration | `python cli.py eval judge-validity` | Sequential stopping with `--sequential` | +| Judge calibration | `python cli.py eval judge validate` | Sequential stopping with `--sequential` | | Tool-choice probe | `python cli.py tools preference` | Cap with `--max-cost-usd` | **Connections.** Every other journey uses these as steps. diff --git a/dokimi/dokimi/caching.py b/dokimi/dokimi/caching.py index 98e779a9e..a6a807e54 100644 --- a/dokimi/dokimi/caching.py +++ b/dokimi/dokimi/caching.py @@ -282,7 +282,7 @@ def is_empty_reply(text: Optional[str]) -> bool: """Whether a reply carries no answer: no content (``None``) or only whitespace. The one test every path shares (response cache, single-turn and session arms, - ``eval reference``, the judge). An empty reply is an error on the cell: it is + ``eval score reference``, the judge). An empty reply is an error on the cell: it is never scored, never counted as a 0, and never stored or replayed. """ return text is None or not text.strip() diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 225112db7..2018c030f 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -126,22 +126,22 @@ def positive_int(value: str) -> int: run.register_compare(sub, positive_int) # compare baseline.register(sub, positive_int) # baseline report.register(sub, positive_int) # report - personas.register(sub, positive_int) # personas - questions.register(sub, positive_int) # questions answers.register(sub, positive_int) # answers - evaluate.register(sub, positive_int) # eval + eval_sub = evaluate.register(sub, positive_int) # eval: the groups below join it (#94 M1) + personas.register(eval_sub, positive_int) # eval personas + questions.register(eval_sub, positive_int) # eval questions, eval import library.register(sub, positive_int) # library benchmark.register(sub, positive_int) # benchmark dashboard.register(sub, positive_int) # dashboard arms.register(sub, positive_int) # arms subjects.register(sub, positive_int) # subjects (scorecards) - tasks.register(sub, positive_int) # tasks - catalog.register(sub, positive_int) # catalog (external skills/tasks) + tasks.register(sub, positive_int, eval_sub) # tasks; eval tasks list, eval inspect + catalog.register(sub, positive_int, eval_sub) # catalog; eval triggers generate explore.register(sub, positive_int) # explore (ExploreBench) dataset.register(sub, positive_int) # dataset ingest.register(sub, positive_int) # ingest novelty.register(sub, positive_int) # novelty (content novelty) - repo.register(sub, positive_int) # repo (repositories as subjects) + repo.register(sub, positive_int, eval_sub) # repo; eval check tools.register(sub, positive_int) # tools (tool-choice probes) context.register(sub, positive_int) # context (context window of a run) runs.register(sub, positive_int) # runs (the artifacts under the runs root) @@ -200,7 +200,7 @@ def main() -> None: args.func(args) except (EmptyQuestionError, InputError, llm_cli.SamplingError) as exc: # bad input, not a bug: no traceback print(f"Error: {exc}", file=sys.stderr) - # 2, as the commands' other input errors: 1 is the verdict of `eval judge-validity` + # 2, as the commands' other input errors: 1 is the verdict of `eval judge validate` # and of the --strict gates, and a malformed file must not read as a failed gate. sys.exit(2) diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 2c1947ed9..ddeedba20 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -36,7 +36,7 @@ class Root: @dataclass(frozen=True) class Leaf: - """An operation: a parser with no subcommands. ``path`` is its argv, e.g. ``("eval", "score")``.""" + """An operation: a parser with no subcommands. ``path`` is its argv, e.g. ``("eval", "score", "pointwise")``.""" path: tuple[str, ...] cost: str @@ -45,7 +45,11 @@ class Leaf: @dataclass(frozen=True) class Tombstone: - """An old argv prefix that now moved: it answers with ``new`` and exits 2, never forwards.""" + """An old argv prefix that now moved: it answers with ``new`` and exits 2, never forwards. + + A group that stays in place (``eval score``) is the exception when no option follows it: see + ``moved_argv``. + """ old: tuple[str, ...] new: tuple[str, ...] @@ -68,8 +72,6 @@ class Tombstone: Root("catalog", "Subjects"), Root("novelty", "Subjects"), Root("repo", "Subjects"), - Root("personas", "Evals"), - Root("questions", "Evals"), Root("eval", "Evals"), Root("dataset", "Analysis and data"), Root("ingest", "Analysis and data"), @@ -100,8 +102,6 @@ class Tombstone: Leaf(("subjects", "show"), FREE), Leaf(("subjects", "run"), PAID), Leaf(("subjects", "evidence"), FREE), - Leaf(("tasks", "list"), FREE), - Leaf(("tasks", "inspect"), FREE), Leaf(("tasks", "harnesses"), FREE), Leaf(("tasks", "run"), PAID), Leaf(("explore", "list"), FREE), @@ -113,7 +113,6 @@ class Tombstone: Leaf(("library", "intents"), FREE), Leaf(("catalog", "scan"), FREE), Leaf(("catalog", "formats"), FREE), - Leaf(("catalog", "trigger-set"), PAID), Leaf(("catalog", "trigger-probe"), PAID), Leaf(("novelty", "analyze"), PAID), Leaf(("novelty", "atoms"), PAID), @@ -128,31 +127,34 @@ class Tombstone: Leaf(("repo", "fetch"), FREE), Leaf(("repo", "reach"), PAID), Leaf(("repo", "budget"), FREE), - Leaf(("repo", "tasks", "check"), FREE), Leaf(("repo", "tasks", "replay"), FREE), Leaf(("repo", "run"), PAID), Leaf(("repo", "report"), FREE), Leaf(("repo", "compare"), FREE), - Leaf(("personas", "discover"), PAID), - Leaf(("personas", "approve"), FREE), - Leaf(("questions", "generate"), PAID), - Leaf(("questions", "analyze"), PAID), - Leaf(("questions", "refine"), PAID), - Leaf(("questions", "panel-review"), PAID), - Leaf(("questions", "import"), FREE), - Leaf(("eval", "score"), PAID), - Leaf(("eval", "ragas"), PAID), - Leaf(("eval", "grounding"), FREE), - Leaf(("eval", "coverage"), FREE), - Leaf(("eval", "judge-validity"), PAID), - Leaf(("eval", "checklist"), PAID), - Leaf(("eval", "judge-wording"), PAID), - Leaf(("eval", "cross-judge"), PAID), - Leaf(("eval", "pairwise"), PAID), - Leaf(("eval", "reference"), PAID), + Leaf(("eval", "inspect"), FREE), + Leaf(("eval", "check"), FREE), + Leaf(("eval", "import"), FREE), Leaf(("eval", "native-import"), FREE), + Leaf(("eval", "tasks", "list"), FREE), + Leaf(("eval", "personas", "discover"), PAID), + Leaf(("eval", "personas", "validate"), FREE), + Leaf(("eval", "questions", "generate"), PAID), + Leaf(("eval", "questions", "analyze"), PAID), + Leaf(("eval", "questions", "refine"), PAID), + Leaf(("eval", "questions", "review"), PAID), + Leaf(("eval", "triggers", "generate"), PAID), + Leaf(("eval", "score", "pointwise"), PAID), + Leaf(("eval", "score", "pairwise"), PAID), + Leaf(("eval", "score", "panel"), PAID), + Leaf(("eval", "score", "checklist"), PAID), + Leaf(("eval", "score", "grounding"), FREE), + Leaf(("eval", "score", "ragas"), PAID), + Leaf(("eval", "score", "reference"), PAID), + Leaf(("eval", "judge", "validate"), PAID), + Leaf(("eval", "judge", "wording"), PAID), + Leaf(("eval", "judge", "agreement"), PAID), + Leaf(("eval", "coverage"), FREE), Leaf(("eval", "redteam"), PAID), - Leaf(("eval", "panel-score"), PAID), Leaf(("dataset", "export"), FREE), Leaf(("ingest", "docling"), FREE), Leaf(("tools", "preference"), PAID), @@ -165,8 +167,33 @@ class Tombstone: Leaf(("data", "schema", "show"), FREE), ) -# No command has moved yet: the renames of #94 §5 M1-M3 add entries here. -TOMBSTONES: tuple[Tombstone, ...] = () +# The moves of #94 §5 M1 (`eval`). M2 and M3 add their entries here. +TOMBSTONES: tuple[Tombstone, ...] = ( + Tombstone(("tasks", "inspect"), ("eval", "inspect")), + Tombstone(("repo", "tasks", "check"), ("eval", "check")), + Tombstone(("personas", "discover"), ("eval", "personas", "discover")), + Tombstone(("personas", "approve"), ("eval", "personas", "validate")), + Tombstone(("questions", "generate"), ("eval", "questions", "generate")), + Tombstone(("questions", "analyze"), ("eval", "questions", "analyze")), + Tombstone(("questions", "refine"), ("eval", "questions", "refine")), + Tombstone(("questions", "panel-review"), ("eval", "questions", "review")), + Tombstone(("questions", "import"), ("eval", "import")), + Tombstone(("tasks", "list"), ("eval", "tasks", "list")), + Tombstone(("catalog", "trigger-set"), ("eval", "triggers", "generate")), + Tombstone(("eval", "score"), ("eval", "score", "pointwise")), + Tombstone(("eval", "pairwise"), ("eval", "score", "pairwise")), + Tombstone(("eval", "panel-score"), ("eval", "score", "panel")), + Tombstone(("eval", "checklist"), ("eval", "score", "checklist")), + Tombstone(("eval", "grounding"), ("eval", "score", "grounding")), + Tombstone(("eval", "ragas"), ("eval", "score", "ragas")), + Tombstone(("eval", "reference"), ("eval", "score", "reference")), + Tombstone(("eval", "judge-validity"), ("eval", "judge", "validate")), + Tombstone(("eval", "judge-wording"), ("eval", "judge", "wording")), + Tombstone(("eval", "cross-judge"), ("eval", "judge", "agreement")), + # The retired roots: a bare `personas` or `questions` (or its help) points at `eval`. + Tombstone(("personas",), ("eval", "personas")), + Tombstone(("questions",), ("eval", "questions")), +) SCHEMA_VERSION = "cli_routes.v1" @@ -206,11 +233,24 @@ def moved_argv( """The argv a tombstone rewrites ``argv`` to, or None when no tombstone matches its start. An alias typed on the command line (``product`` for ``library``) matches the root's tombstones. + A path that is itself a live leaf under the old prefix is not a tombstone: ``eval score + pointwise`` is live even though ``eval score`` moved there. """ canon = _canonical(tuple(argv)) + live = {leaf.path for leaf in LEAVES} for tomb in TOMBSTONES if tombstones is None else tombstones: - if canon[: len(tomb.old)] == _canonical(tomb.old): - return [*tomb.new, *argv[len(tomb.old) :]] + n = len(tomb.old) + if canon[:n] != _canonical(tomb.old): + continue + if any(canon[:k] in live for k in range(n + 1, len(canon) + 1)): + continue + # A group that stays under its own name (`eval score` -> `eval score pointwise`) keeps its help. + # A plain word after it is a mistyped leaf (argparse names the choices); an option is an old call. + if tomb.new[:n] == _canonical(tomb.old) and ( + all(a in ("-h", "--help") for a in argv[n:]) or not argv[n].startswith("-") + ): + continue + return [*tomb.new, *argv[n:]] return None @@ -231,7 +271,8 @@ def _resolve(path: tuple[str, ...]) -> tuple[str, ...] | None: def _is_route(path: tuple[str, ...]) -> bool: """True when argv ``path`` names a root or an alias, or a group or leaf under one. - ``moved_argv`` matches by prefix, so a tombstone on a group shadows every route under it. + ``moved_argv`` matches by prefix and skips a live leaf, so a tombstone on a group shadows + the argv under it that is not a live leaf. """ full = _resolve(path) return full is not None and any(leaf.path[: len(full)] == full for leaf in LEAVES) @@ -251,7 +292,10 @@ def tombstone_problems(tombstones: tuple[Tombstone, ...] | None = None) -> list[ for tomb in TOMBSTONES if tombstones is None else tombstones: if not tomb.old or tomb.old[0].startswith("-"): # an empty path matches every argv problems.append(f"{' '.join(tomb.old) or '(empty)'} is not a command path") - if _is_route(tomb.old): + # A group the move leaves in place (`eval score`) is a route, but not an old one. + # Only a group of two or more words can stay in place; a whole root or alias cannot. + stays = len(tomb.old) > 1 and tomb.new[: len(tomb.old)] == _canonical(tomb.old) + if _is_route(tomb.old) and not stays: problems.append(f"{' '.join(tomb.old)} is still a live route") if _extends_a_leaf(tomb.old): problems.append(f"{' '.join(tomb.old)} extends a live route and would shadow it") diff --git a/dokimi/dokimi/commands/answer_formats.py b/dokimi/dokimi/commands/answer_formats.py index 7c808355b..cb2ef4e22 100644 --- a/dokimi/dokimi/commands/answer_formats.py +++ b/dokimi/dokimi/commands/answer_formats.py @@ -1,11 +1,11 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""`questions import`, `eval reference` and `eval native-import`: answer-only eval formats. +"""`eval import`, `eval score reference` and `eval native-import`: answer-only eval formats. Inspect AI, OpenAI evals, skill-creator evals.json and promptfoo sets are imported as questions.v2 with ``grading`` checks, answered by the answer track (``answers generate`` or -``arms run``) and scored by ``eval reference``. A native tool's own log is read by +``arms run``) and scored by ``eval score reference``. A native tool's own log is read by ``eval native-import`` so its completions and verdicts sit next to dokimi's. """ @@ -88,12 +88,12 @@ def cmd_eval_native_import(args: argparse.Namespace) -> None: print(f"{run.tool} ({run.model}): {len(run.answers)} samples, native pass {rate}") -def register_questions(questions_sub) -> None: - p = questions_sub.add_parser( +def register_import(eval_sub) -> None: + p = eval_sub.add_parser( "import", help="Import an answer-only eval (Inspect samples/@task, OpenAI evals, skill-creator, promptfoo) " "as questions with grading checks") - p.add_argument("path", help="Eval file (format detected as by `tasks inspect`)") + p.add_argument("path", help="Eval file (format detected as by `eval inspect`)") p.add_argument("--format", default=None, help="Force a format name (default: detected)") p.add_argument("--scorer", default=None, help="Inspect samples: scorer to translate (match, choice, ...)") p.add_argument("--task", default=None, help="Inspect @task file: which task function") @@ -103,14 +103,14 @@ def register_questions(questions_sub) -> None: p.set_defaults(func=cmd_questions_import) -def register_eval(eval_sub, positive_int) -> None: +def register_eval(score_sub, eval_sub, positive_int) -> None: from dokimi.defaults import DEFAULT_JUDGE_PROVIDER, JUDGE_MODEL_FOR_PROVIDER, JUDGE_MODEL_HELP from dokimi.eval.judge import JUDGE_PROVIDERS - r = eval_sub.add_parser("reference", help="Score answers against imported grading checks " + r = score_sub.add_parser("reference", help="Score answers against imported grading checks " "(match/exact/choice/... deterministic, judge checks via the judge)") r.add_argument("--answers", required=True, help="answers.v1 or arms output JSON") - r.add_argument("--questions", required=True, help="Questions JSON from `questions import`") + r.add_argument("--questions", required=True, help="Questions JSON from `eval import`") r.add_argument("-o", "--output", required=True, help="eval.v1 (answers) or arms (arms run) JSON to write") r.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, help=JUDGE_MODEL_HELP) r.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index ecb1401f0..a909f70d5 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -98,7 +98,7 @@ def cmd_answers_generate(args: argparse.Namespace) -> None: print(f"Documentation source: {doc_source}") context7_id = getattr(args, "context7_id", None) if doc_source == "none": - # Baseline only: imported evals (`questions import`) are asked without doc context. + # Baseline only: imported evals (`eval import`) are asked without doc context. mcp_client, library_id = None, args.product else: mcp_client = create_doc_source_client(doc_source) diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 3890cbd01..61e6c561e 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -490,7 +490,7 @@ def _load_questions(path: str) -> list[dict]: def _refuse_external_trigger_set(path: str) -> None: """Point a skill-reach or skill-creator trigger set at the command that reads it. - Those sets are read through ``catalog trigger-set --trigger-set``, which also + Those sets are read through ``eval triggers generate --trigger-set``, which also lints them against the catalog the arm will offer; a questions loader that guessed at them would drop that lint and, for skill-creator, the target skill. """ @@ -507,7 +507,7 @@ def _refuse_external_trigger_set(path: str) -> None: target = "=" if fmt == "skill-creator" else "" raise ValueError( f"{path} is a {fmt} trigger set, not a questions set; convert it with " - f"`dokimi catalog trigger-set --catalog --trigger-set {path}{target} " + f"`dokimi eval triggers generate --catalog --trigger-set {path}{target} " f"--format questions --out q.json`, then run the skill-catalog: arm on q.json") diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index fa7cfb29a..13a80524c 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -314,8 +314,8 @@ def _trigger_set_sources(specs: list, catalog) -> list: return out -def register(sub, positive_int) -> None: - """Add the `catalog` subcommand group.""" +def register(sub, positive_int, eval_sub) -> None: + """Add the `catalog` subcommand group, and `eval triggers generate` under `eval` (#94 M1).""" catalog_p = sub.add_parser( "catalog", @@ -359,8 +359,10 @@ def register(sub, positive_int) -> None: fm_p.add_argument("--json", action="store_true", help="Print JSON instead of a table") fm_p.set_defaults(func=cmd_catalog_formats) - ts_p = catalog_sub.add_parser( - "trigger-set", + triggers_sub = eval_sub.add_parser("triggers", help="Trigger sets: build them from question sets").add_subparsers( + dest="eval_triggers_cmd", required=True) + ts_p = triggers_sub.add_parser( + "generate", help="Build a skill-selection query set (labels + leakage/framing lint) from question sets", ) ts_p.add_argument("--catalog", required=True, help="Directory of the skills the agent will see") diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index b1b911fd1..fa8665d25 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""eval subcommand group: score, ragas, panel-score.""" +"""eval subcommand group: score, judge, and the eval sets (personas, questions, tasks, triggers, check).""" from __future__ import annotations @@ -145,7 +145,7 @@ def cmd_eval_panel_score(args: argparse.Namespace) -> None: answer_model=answer_model, judge_provider=cfg.provider, judge_model=cfg.model, - context="panel-score", + context="eval score panel", ) n_effective = min(len(answers), args.limit) if args.limit else len(answers) @@ -197,7 +197,7 @@ def cmd_eval_score(args: argparse.Namespace) -> None: answer_model=answer_model, judge_provider=args.judge_provider, judge_model=args.judge_model, - context="eval-score", + context="eval score pointwise", ) try: @@ -472,15 +472,22 @@ def _non_negative_int(value: str) -> int: return n -def register(sub, positive_int) -> None: - """Add the `eval` subcommand group.""" +def register(sub, positive_int): + """Add the `eval` subcommand group and its `score` and `judge` groups; return the eval subparsers. + + The groups that join `eval` (personas, questions, tasks, triggers, check) register on the return value. + """ # Eval subcommand group - eval_p = sub.add_parser("eval", help="Evaluate answers") + eval_p = sub.add_parser("eval", help="Build and check eval sets, score answers, calibrate judges") eval_sub = eval_p.add_subparsers(dest="eval_cmd", required=True) eval_coverage.register(eval_sub) - # eval score - score_p = eval_sub.add_parser("score", help="Score answers using LLM-as-judge") + score_group = eval_sub.add_parser( + "score", help="Score answers: pointwise, pairwise, panel, checklist, grounding, ragas, reference") + score_sub = score_group.add_subparsers(dest="score_cmd", required=True) + + # eval score pointwise + score_p = score_sub.add_parser("pointwise", help="Score answers using LLM-as-judge") score_p.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") score_p.add_argument("--answers", required=True, help="Path to answers JSON file") score_p.add_argument("--run-id", default=None, dest="run_id", @@ -495,8 +502,8 @@ def register(sub, positive_int) -> None: help="Judge the no-retrieval placeholder answers too (default: they are not scored)") score_p.set_defaults(func=cmd_eval_score) - # eval ragas - ragas_p = eval_sub.add_parser("ragas", help="Run RAGAS meta-evaluation on answers") + # eval score ragas + ragas_p = score_sub.add_parser("ragas", help="Run RAGAS meta-evaluation on answers") ragas_p.add_argument("--answers", required=True, help="Path to answers JSON file") ragas_p.add_argument("--ragas-model", default=DEFAULT_JUDGE_MODEL, help=f"LLM for RAGAS judge (default: {DEFAULT_JUDGE_MODEL})") @@ -504,8 +511,8 @@ def register(sub, positive_int) -> None: help=f"Provider for RAGAS judge LLM (default: {DEFAULT_JUDGE_PROVIDER})") ragas_p.set_defaults(func=cmd_eval_ragas) - # eval grounding (reference-free, no LLM) - grounding_p = eval_sub.add_parser( + # eval score grounding (reference-free, no LLM) + grounding_p = score_sub.add_parser( "grounding", help="Reference-free grounding/citation metrics with confidence intervals" ) grounding_p.add_argument("--answers", required=True, help="Path to answers JSON file") @@ -514,9 +521,11 @@ def register(sub, positive_int) -> None: help="Grounding score above which an answer counts as grounded (default: 0.5)") grounding_p.set_defaults(func=cmd_eval_grounding) - # eval judge-validity (does the judge order known-worse answers below better ones?) - validity_p = eval_sub.add_parser( - "judge-validity", + # eval judge validate (does the judge order known-worse answers below better ones?) + judge_group = eval_sub.add_parser("judge", help="Judge calibration: validate, wording, agreement") + judge_sub = judge_group.add_subparsers(dest="judge_cmd", required=True) + validity_p = judge_sub.add_parser( + "validate", help="Measure judge discriminative validity on deterministic degradations (exits 1 on FAIL)", ) validity_p.add_argument("--answers", required=True, help="Path to answers JSON file") @@ -574,18 +583,18 @@ def register(sub, positive_int) -> None: from dokimi.commands import judge_reliability - judge_reliability.register(eval_sub, positive_int) # eval checklist, eval judge-wording + judge_reliability.register(score_sub, judge_sub, positive_int) # eval score checklist and pairwise; eval judge wording and agreement from dokimi.commands import answer_formats - answer_formats.register_eval(eval_sub, positive_int) # eval reference, eval native-import + answer_formats.register_eval(score_sub, eval_sub, positive_int) # eval score reference; eval native-import from dokimi.commands import redteam redteam.register_eval(eval_sub) # eval redteam - # eval panel-score - panel_p = eval_sub.add_parser("panel-score", help="Score answers using a multi-judge panel") + # eval score panel + panel_p = score_sub.add_parser("panel", help="Score answers using a multi-judge panel") panel_p.add_argument("--answers", required=True, help="Path to answers JSON file") panel_p.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") panel_p.add_argument("--output", default=None, help="Output file (default: eval/{product}_panel.json)") @@ -610,3 +619,5 @@ def register(sub, positive_int) -> None: panel_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", help="Sampling temperature for every panel judge, recorded as judge_temperature") panel_p.set_defaults(func=cmd_eval_panel_score) + + return eval_sub diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index e78d07959..931b8797c 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""eval checklist / eval judge-wording: the judge-reliability checks beyond judge-validity.""" +"""eval score checklist / eval judge wording: the judge-reliability checks beyond judge-validity.""" from __future__ import annotations @@ -57,7 +57,7 @@ def _judge_name(doc: dict, path: str) -> str: def _result_measure(path: str, key: str, kind: type, build): - """The measure an `eval pairwise` or `eval checklist` file gives, or exit 2 naming the file. + """The measure an `eval score pairwise` or `eval score checklist` file gives, or exit 2 naming the file. *build* is ``(doc, name) -> Measure``; it raises ValueError on an entry of the wrong shape. """ @@ -103,7 +103,7 @@ def _require_arms(path: str, records, command: str) -> None: if not has_arms(records): print(f"Error: {path} has no arms answers (no record has an `arms` field); " - f"eval {command} reads an arms run (arms run --out-json), not a binary " + f"{command} reads an arms run (arms run --out-json), not a binary " "with_docs/without_docs answers file", file=sys.stderr) sys.exit(2) @@ -180,7 +180,7 @@ def cmd_eval_pairwise(args: argparse.Namespace) -> None: baseline = args.baseline_arm or (data.get("baseline_arm") if isinstance(data, dict) else None) or "baseline" library = args.product or (data.get("library_name") if isinstance(data, dict) else None) or "the library" arms = [a.strip() for a in args.arms.split(",")] if args.arms else None - _require_arms(args.answers, records, "pairwise") + _require_arms(args.answers, records, "eval score pairwise") pairwise_jobs(records, baseline_arm=baseline, arms=arms) # refuse a blank question before the probe call judge, label = pairwise_judge(args) print(f"Pairwise comparison: {label}, {len(records)} questions, baseline `{baseline}`") @@ -214,7 +214,7 @@ def cmd_eval_checklist(args: argparse.Namespace) -> None: points = _load_points(args.questions) baseline = args.baseline_arm or (data.get("baseline_arm") if isinstance(data, dict) else None) or "baseline" arms = [a.strip() for a in args.arms.split(",")] if args.arms else None - _require_arms(args.answers, records, "checklist") + _require_arms(args.answers, records, "eval score checklist") checklist_jobs(records, points, arms=arms) # refuse a blank question before the probe call _check_judge_temperature([(args.judge_model, args.judge_provider)], args.judge_temperature) llm = Judge(model=args.judge_model, provider=args.judge_provider, temperature=args.judge_temperature).llm @@ -407,7 +407,7 @@ def score(library, question, answer, ground_truth, context): validity = [_load_json_object(v, "summary")["summary"] for v in args.validity or []] specs = [j.strip() for j in (args.judges or "").split(",") if j.strip()] if specs: - _require_arms(args.answers, records, "cross-judge --judges") + _require_arms(args.answers, records, "eval judge agreement --judges") scored_answers(records) # refuse a blank question before the probe call _check_judge_temperature([(m, p) for p, _, m in (s.partition("/") for s in specs) if m and p in JUDGE_PROVIDERS], args.judge_temperature) @@ -477,9 +477,9 @@ def _strict_cross_judge_exit(arms: dict[str, dict[str, Any]], baseline_arm: str) sys.exit(1) -def register(eval_sub, positive_int) -> None: - """Add `eval checklist`, `eval judge-wording` and `eval cross-judge` to the eval group.""" - ck = eval_sub.add_parser( +def register(score_sub, judge_sub, positive_int) -> None: + """Add `eval score checklist` and `eval score pairwise`, and `eval judge wording` and `eval judge agreement`.""" + ck = score_sub.add_parser( "checklist", help="Required-point coverage per arm: is each expected_topic in the answer? (both polarities)", ) @@ -497,8 +497,8 @@ def register(eval_sub, positive_int) -> None: help="Output JSON (default: _checklist.json); Markdown written next to it") ck.set_defaults(func=cmd_eval_checklist) - jw = eval_sub.add_parser( - "judge-wording", + jw = judge_sub.add_parser( + "wording", help="Judge-validity under rewrites of the judge prompt, paired against the original on the same pairs", ) jw.add_argument("--answers", required=True, help="Path to answers JSON file") @@ -521,8 +521,8 @@ def register(eval_sub, positive_int) -> None: jw.add_argument("--output", default=None, help="Output JSON report path") jw.set_defaults(func=cmd_eval_judge_wording) - cj = eval_sub.add_parser( - "cross-judge", + cj = judge_sub.add_parser( + "agreement", help="Each arm's delta under several judges and axes, labelled CONSISTENT / JUDGE-DEPENDENT / ...", ) cj.add_argument("--answers", required=True, help="Arms output JSON (arms run --out-json)") @@ -530,11 +530,11 @@ def register(eval_sub, positive_int) -> None: help="Comma-separated provider/model judges to re-score with, e.g. " "amazon-bedrock/us.anthropic.claude-haiku-5-5,amazon-bedrock/openai.gpt-oss-120b-1:0") cj.add_argument("--checklist", action="append", default=None, - help="An `eval checklist` JSON to include as a coverage measure (repeatable)") + help="An `eval score checklist` JSON to include as a coverage measure (repeatable)") cj.add_argument("--pairwise", action="append", default=None, - help="An `eval pairwise` JSON to include as a measure (repeatable)") + help="An `eval score pairwise` JSON to include as a measure (repeatable)") cj.add_argument("--validity", action="append", default=None, - help="An `eval judge-validity` JSON; that judge's measures are marked PASS/FAIL and a " + help="An `eval judge validate` JSON; that judge's measures are marked PASS/FAIL and a " "FAILed judge does not vote in the verdict (repeatable)") cj.add_argument("--require-validated", action="store_true", dest="require_validated", help="Only judges that PASSED a --validity gate vote in the verdict") @@ -556,7 +556,7 @@ def register(eval_sub, positive_int) -> None: help="Output JSON (default: _crossjudge.json); Markdown written next to it") cj.set_defaults(func=cmd_eval_cross_judge) - pw = eval_sub.add_parser( + pw = score_sub.add_parser( "pairwise", help="Every arm vs the baseline, both answers shown together in both orders; optional escalation", ) diff --git a/dokimi/dokimi/commands/personas.py b/dokimi/dokimi/commands/personas.py index 311f0d00a..715ac0900 100644 --- a/dokimi/dokimi/commands/personas.py +++ b/dokimi/dokimi/commands/personas.py @@ -72,7 +72,7 @@ def cmd_personas_discover(args: argparse.Namespace) -> None: print("\nNext steps:") print(f" 1. Review: cat {output_path}") print(" 2. Edit if needed") - print(f" 3. Approve: dokimi personas approve --file {output_path}") + print(f" 3. Approve: dokimi eval personas validate --file {output_path}") def cmd_personas_approve(args: argparse.Namespace) -> None: @@ -119,15 +119,15 @@ def cmd_personas_approve(args: argparse.Namespace) -> None: print(f"✓ Validated {len(personas['personas'])} personas") print(f"✓ File approved: {persona_file}") print("\nReady for question generation:") - print(f" dokimi questions generate --product {personas['product']} --personas {persona_file}") + print(f" dokimi eval questions generate --product {personas['product']} --personas {persona_file}") -def register(sub, positive_int) -> None: - """Add the `personas` subcommand group.""" - personas_p = sub.add_parser("personas", help="Persona discovery and management") +def register(eval_sub, positive_int) -> None: + """Add `eval personas discover` and `eval personas validate` (#94 M1).""" + personas_p = eval_sub.add_parser("personas", help="Persona discovery and management") personas_sub = personas_p.add_subparsers(dest="personas_cmd", required=True) - # personas discover + # eval personas discover discover_p = personas_sub.add_parser("discover", help="Discover personas from a GitHub repo or product description") discover_p.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") discover_p.add_argument("--repo", default=None, help="GitHub repo (e.g., uxlfoundation/oneTBB). Optional if --description is given.") @@ -140,7 +140,7 @@ def register(sub, positive_int) -> None: discover_p.add_argument("--save-analysis", action="store_true", help="Save intermediate analysis JSON") discover_p.set_defaults(func=cmd_personas_discover) - # personas approve - approve_p = personas_sub.add_parser("approve", help="Validate and approve persona file") + # eval personas validate + approve_p = personas_sub.add_parser("validate", help="Validate and approve persona file") approve_p.add_argument("--file", required=True, help="Persona JSON file to approve") approve_p.set_defaults(func=cmd_personas_approve) diff --git a/dokimi/dokimi/commands/questions.py b/dokimi/dokimi/commands/questions.py index 4387c05d9..9cbcc9ada 100644 --- a/dokimi/dokimi/commands/questions.py +++ b/dokimi/dokimi/commands/questions.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""questions subcommand group: generate, analyze, refine, panel-review.""" +"""questions subcommand group: generate, analyze, refine, review (was panel-review).""" from __future__ import annotations @@ -374,9 +374,9 @@ def cmd_questions_generate(args: argparse.Namespace) -> None: print(f"\n✅ Saved {len(questions)} questions to {output_path}") -def register(sub, positive_int) -> None: - """Add the `questions` subcommand group.""" - questions_p = sub.add_parser("questions", help="Question generation") +def register(eval_sub, positive_int) -> None: + """Add the `eval questions` group and `eval import` (#94 M1).""" + questions_p = eval_sub.add_parser("questions", help="Question generation") questions_sub = questions_p.add_subparsers(dest="questions_cmd", required=True) # questions generate @@ -429,8 +429,8 @@ def register(sub, positive_int) -> None: choices=["openai", "anthropic", "google"]) refine_q_p.set_defaults(func=cmd_questions_refine) - # questions panel-review - panel_q_p = questions_sub.add_parser("panel-review", + # questions review (was panel-review) + panel_q_p = questions_sub.add_parser("review", help="Multi-agent panel review of question quality") panel_q_p.add_argument("--questions", required=True, help="Path to questions JSON file") panel_q_p.add_argument("--product", required=True, help="Product name (e.g., oneDAL)") @@ -447,4 +447,4 @@ def register(sub, positive_int) -> None: from dokimi.commands import answer_formats - answer_formats.register_questions(questions_sub) # questions import + answer_formats.register_import(eval_sub) # eval import diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 88f64ad09..3d6bf3f4c 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -421,12 +421,12 @@ def _cache_dir(args: argparse.Namespace) -> Path | None: return Path(args.cache_dir) if args.cache_dir else None -def register(sub, positive_int) -> None: - """Add the `repo` subcommand group.""" +def register(sub, positive_int, eval_sub) -> None: + """Add the `repo` subcommand group. Its check moved to `eval check` (#94 M1).""" repo_p = sub.add_parser( "repo", help="Repositories as evaluation subjects: validate repo-eval.yaml, L0 scan, fetch sources, reach probe, budget, " - "task self-check, agent runs and their report", + "agent runs and their report", ) repo_sub = repo_p.add_subparsers(dest="repo_cmd", required=True) source_help = ( @@ -563,4 +563,4 @@ def register(sub, positive_int) -> None: from . import repo_run - repo_run.register(repo_sub, positive_int, source_help, cache_help) + repo_run.register(repo_sub, eval_sub, positive_int, source_help, cache_help) diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 8c81f0859..f3afff139 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""repo tasks check / repo tasks replay / repo run / repo report / repo compare: the repository-mode runner +"""eval check / repo tasks replay / repo run / repo report / repo compare: the repository-mode runner (contract §3, §4, §4.3, §4.5).""" from __future__ import annotations @@ -23,7 +23,7 @@ def _context(args: argparse.Namespace, out: Path, pin: bool = False): - """Source, manifest, exported eval dir, image and caches shared by `tasks check` and `run`. + """Source, manifest, exported eval dir, image and caches shared by `eval check` and `run`. With ``pin`` (a batch), the batch keeps the eval tree it started with: a later invocation from another revision would grade new cells with other graders, so it is refused. A working @@ -408,10 +408,11 @@ def _common(p: argparse.ArgumentParser, source_help: str, cache_help: str, posit p.add_argument("--cache-dir", default=None, dest="cache_dir", help=cache_help) -def register(repo_sub, positive_int, source_help: str, cache_help: str) -> None: - tasks_p = repo_sub.add_parser("tasks", help="Task self-checks, and review tasks built from past reviews (no LLM)") +def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str) -> None: + tasks_p = repo_sub.add_parser("tasks", help="Review tasks built from past reviews (no LLM)") tasks_sub = tasks_p.add_subparsers(dest="repo_tasks_cmd", required=True) - check_p = tasks_sub.add_parser( + # eval check (#94 M1): the self-check that `repo tasks check` ran, now `eval check` + check_p = eval_sub.add_parser( "check", help="Per task: grade on the untouched workspace must fail the gate, oracle then grade must " "pass it (score-only: reach score.max), and the agent's container must not see tasks or source") _common(check_p, source_help, cache_help, positive_int) diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py index cf04ea8c3..f9f201f61 100644 --- a/dokimi/dokimi/commands/tasks.py +++ b/dokimi/dokimi/commands/tasks.py @@ -249,8 +249,8 @@ def cmd_tasks_run(args: argparse.Namespace) -> None: _print_summary(output) -def register(sub, positive_int) -> None: - """Add the `tasks` subcommand group.""" +def register(sub, positive_int, eval_sub) -> None: + """Add the `tasks` subcommand group (run, harnesses) and, under `eval`, `tasks list` and `inspect`.""" tasks_p = sub.add_parser( "tasks", @@ -258,11 +258,15 @@ def register(sub, positive_int) -> None: ) tasks_sub = tasks_p.add_subparsers(dest="tasks_cmd", required=True) - list_p = tasks_sub.add_parser("list", help="List terminal-bench task metadata") + # eval tasks list (#94 M1) + eval_tasks_sub = eval_sub.add_parser("tasks", help="Eval tasks: list them").add_subparsers( + dest="eval_tasks_cmd", required=True) + list_p = eval_tasks_sub.add_parser("list", help="List terminal-bench task metadata") list_p.add_argument("--tasks-root", default=None, help=_TASKS_ROOT_HELP) list_p.set_defaults(func=cmd_tasks_list) - inspect_p = tasks_sub.add_parser( + # eval inspect (#94 M1) + inspect_p = eval_sub.add_parser( "inspect", help="Detect the eval-task format of a file or directory tree and report its structure (no run)", ) diff --git a/dokimi/dokimi/eval/cross_judge.py b/dokimi/dokimi/eval/cross_judge.py index 0ff1c923a..6eeae7fd3 100644 --- a/dokimi/dokimi/eval/cross_judge.py +++ b/dokimi/dokimi/eval/cross_judge.py @@ -37,7 +37,7 @@ too. A judge's verdict should only count if the judge can order answers whose order -is known. Pass ``eval judge-validity`` reports with :meth:`CrossJudgeReport. +is known. Pass ``eval judge validate`` reports with :meth:`CrossJudgeReport. apply_validity` and every measure from a judge that FAILED its gate — its grounded scoring included — is shown but left out of the label. With ``require_validated`` only judges that PASSED vote. On the Opus-answer perf run @@ -212,11 +212,11 @@ def unreproducible_arms(data: Dict[str, Any]) -> List[str]: This is the gate for pairwise and checklist judges, which are shown the answers alone, so every arm the stored judge saw material for is unreproducible. A - re-scored judge (``cross-judge --judges``) is gated per question instead, on + re-scored judge (``eval judge agreement --judges``) is gated per question instead, on what :meth:`JudgeContextStore.from_artifact` serves it. The arms run stores its judge's view of treatment material in ``judge_materials`` - (:mod:`dokimi.eval.judge_context`). ``cross-judge --judges`` gives its judge + (:mod:`dokimi.eval.judge_context`). ``eval judge agreement --judges`` gives its judge that same material by using the artifact's ``judge_materials``. Pairwise and checklist judges see the answers alone. In review, the same judge on the same answers read +24.2 (p 0.0006) with the full material (11,079 chars) and +1.4 @@ -268,7 +268,7 @@ def _number(x: Any) -> float: def coverage_measure(checklist: Dict[str, Any], name: str) -> Measure: - """An `eval checklist` report as a measure: each arm scores its coverage. + """An `eval score checklist` report as a measure: each arm scores its coverage. Raises ValueError naming the first question that is not ``{question_id, arms: {arm: {coverage}}}`` with a finite number or null as the coverage. @@ -289,7 +289,7 @@ def coverage_measure(checklist: Dict[str, Any], name: str) -> Measure: def pairwise_measure(pairwise: Dict[str, Any], name: str, baseline_arm: str) -> Measure: - """An `eval pairwise` report as a measure: the arm scores its verdict, the baseline 0. + """An `eval score pairwise` report as a measure: the arm scores its verdict, the baseline 0. Raises ValueError naming the first verdict that is not an object with a finite number as its ``score``. @@ -413,7 +413,7 @@ class CrossJudgeReport: def apply_validity(self, validity: Sequence[Dict[str, Any]]) -> List[str]: """Mark measures with their judge's gate result; returns validity labels that matched nothing. - ``validity`` holds ``eval judge-validity`` summaries. A PASS marks the + ``validity`` holds ``eval judge validate`` summaries. A PASS marks the absolute or pairwise measure of the same judge label and the same context (``none`` or ``stored``; a report without ``context`` counts as ``none``): the gate only validated the setting it ran in. A FAIL reaches @@ -474,7 +474,7 @@ def format_markdown(self) -> str: "## Verdict robustness", "", f"Every arm against `{s['baseline_arm']}`, under each judge and axis. A verdict that holds under " - "one judge only is a property of that judge. Gate: the judge's own `judge-validity` result; " + "one judge only is a property of that judge. Gate: the judge's own `eval judge validate` result; " "a measure whose judge failed it (✗) is shown but does not vote" + (", and neither does an unvalidated one (—)." if self.require_validated else "."), "", diff --git a/dokimi/dokimi/eval/judge_context.py b/dokimi/dokimi/eval/judge_context.py index 55be99805..488bfd7b4 100644 --- a/dokimi/dokimi/eval/judge_context.py +++ b/dokimi/dokimi/eval/judge_context.py @@ -249,7 +249,7 @@ def from_artifact(cls, data: Dict[str, Any]) -> "JudgeContextStore": logger.warning( "%d arm records cite judge material that is missing from the artifact " "or does not match its digest; those answers have no verified material " - "(callers that fall back use the report snippets; judge-validity " + "(callers that fall back use the report snippets; eval judge validate " "--context stored drops them).", bad, ) return store @@ -266,7 +266,7 @@ def get(self, question_id: Union[str, int], arm_name: str) -> Optional[str]: "No stored judge context for %s/%s (the run did not keep it, or the text " "failed its digest). Callers that fall back judge the 200-char report " "snippets, which under-scores arms that cite their own material; " - "judge-validity --context stored drops such bases instead.", + "eval judge validate --context stored drops such bases instead.", question_id, arm_name, ) return hit diff --git a/dokimi/dokimi/eval/pairwise.py b/dokimi/dokimi/eval/pairwise.py index 1653a384a..a3611cb11 100644 --- a/dokimi/dokimi/eval/pairwise.py +++ b/dokimi/dokimi/eval/pairwise.py @@ -45,7 +45,7 @@ Not every model is good at this. Pairwise Haiku 4.5 preferred the self-contradicting copy outright on 27 of 40 pairs and tied 13 (0.16 on -``contradiction``), so a pairwise judge must pass ``eval judge-validity --mode +``contradiction``), so a pairwise judge must pass ``eval judge validate --mode pairwise`` like any other. """ diff --git a/dokimi/dokimi/eval/question_text.py b/dokimi/dokimi/eval/question_text.py index 6ef8c454a..eeca1bb5c 100644 --- a/dokimi/dokimi/eval/question_text.py +++ b/dokimi/dokimi/eval/question_text.py @@ -29,7 +29,7 @@ class EmptyQuestionError(ValueError): A ``ValueError`` meaning a bad input record, which the CLI reports as ``Error: ...`` with exit 2, the code the commands use for bad input (1 is - the verdict code of ``--strict`` gates and ``eval judge-validity``). + the verdict code of ``--strict`` gates and ``eval judge validate``). """ diff --git a/dokimi/dokimi/eval/reference_run.py b/dokimi/dokimi/eval/reference_run.py index 9c3a81779..8e7289cc1 100644 --- a/dokimi/dokimi/eval/reference_run.py +++ b/dokimi/dokimi/eval/reference_run.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""``eval reference``: grade an answers or arms file with its questions' ``grading`` checks. +"""``eval score reference``: grade an answers or arms file with its questions' ``grading`` checks. The answers come from the usual answer track (``answers generate``, ``arms run``) or from a native tool's log (:mod:`dokimi.task_formats.imports.native_logs`); the checks are diff --git a/dokimi/dokimi/questions/ragas_seed.py b/dokimi/dokimi/questions/ragas_seed.py index 0ce588e4b..078c16ce5 100644 --- a/dokimi/dokimi/questions/ragas_seed.py +++ b/dokimi/dokimi/questions/ragas_seed.py @@ -185,7 +185,7 @@ def _fallback_topics(library_name: str) -> List[str]: """Return generic fallback topics when extraction fails. Product-specific topic lists are evaluation content: supply them with - ``questions generate --topics `` instead of hard-coding them + ``eval questions generate --topics `` instead of hard-coding them here, so the fallback is the same for every library. """ return ["installation", "getting started", "API overview", diff --git a/dokimi/dokimi/repos/host.py b/dokimi/dokimi/repos/host.py index d504e7eda..2789a0bbe 100644 --- a/dokimi/dokimi/repos/host.py +++ b/dokimi/dokimi/repos/host.py @@ -17,7 +17,7 @@ started like the grading container (same image, same runner flags). Its CPU count is the affinity mask capped by the cgroup CPU quota (``docker --cpus``), which is what a threading runtime such as oneTBB sizes itself by. The probe -runs once per ``repo run`` or ``repo tasks check`` invocation; its facts are +runs once per ``repo run`` or ``eval check`` invocation; its facts are recorded with the batch and in each cell's ``meta.json``. """ diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index 3e3a1a782..b0a2fce4d 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""``repo tasks check``: the self-check every task passes before any paid run (contract §4.3). No LLM. +"""``eval check``: the self-check every task passes before any paid run (contract §4.3). No LLM. Per task, on workspaces prepared exactly as for an agent (arm ``guidance:raw``, empty ``answer.txt`` and ``events.jsonl``): diff --git a/dokimi/dokimi/schemas/questions.v2.json b/dokimi/dokimi/schemas/questions.v2.json index 7b2606e69..a4384d712 100644 --- a/dokimi/dokimi/schemas/questions.v2.json +++ b/dokimi/dokimi/schemas/questions.v2.json @@ -12,7 +12,7 @@ "const": "questions.v2" }, "library": { - "description": "Product the set is for (curated sets, `questions refine`).", + "description": "Product the set is for (curated sets, `eval questions refine`).", "type": "string" }, "library_id": { @@ -32,29 +32,29 @@ "type": "string" }, "generated_at": { - "description": "UTC timestamp the generator wrote the set (`questions generate`).", + "description": "UTC timestamp the generator wrote the set (`eval questions generate`).", "type": "string" }, "model": { - "description": "Generator model (`questions generate`).", + "description": "Generator model (`eval questions generate`).", "type": "string" }, "provider": { - "description": "Generator provider (`questions generate`).", + "description": "Generator provider (`eval questions generate`).", "type": "string" }, "total_questions": { - "description": "Number of questions in the set (`questions generate`, pipeline).", + "description": "Number of questions in the set (`eval questions generate`, pipeline).", "type": "integer", "minimum": 0 }, "total": { - "description": "Number of questions in the set (`questions refine`).", + "description": "Number of questions in the set (`eval questions refine`).", "type": "integer", "minimum": 0 }, "difficulty_distribution": { - "description": "Question count per normalized difficulty label after refinement (`questions refine`).", + "description": "Question count per normalized difficulty label after refinement (`eval questions refine`).", "type": "object", "additionalProperties": { "type": "integer", @@ -174,7 +174,7 @@ } }, "expected_skill": { - "description": "Skill (or list of skills) the question is meant to exercise; null when none should apply (curated sets, `catalog trigger-set`).", + "description": "Skill (or list of skills) the question is meant to exercise; null when none should apply (curated sets, `eval triggers generate`).", "type": [ "string", "array", @@ -189,11 +189,11 @@ "type": "object" }, "prompt_style": { - "description": "`verbatim`: the answerer sends `question` as written, without its expert-answer wrapper (set by `questions import`, whose graders score the reply to the original prompt).", + "description": "`verbatim`: the answerer sends `question` as written, without its expert-answer wrapper (set by `eval import`, whose graders score the reply to the original prompt).", "type": "string" }, "grading": { - "description": "Format-neutral reference checks from an imported eval (`questions import`): `checks` (match, exact, fuzzy, regex, choice, json, judge, unsupported), optional `threshold`, `source_format`, `native_scorer`. Scored by `eval reference`; see dokimi.eval.reference_checks.", + "description": "Format-neutral reference checks from an imported eval (`eval import`): `checks` (match, exact, fuzzy, regex, choice, json, judge, unsupported), optional `threshold`, `source_format`, `native_scorer`. Scored by `eval score reference`; see dokimi.eval.reference_checks.", "type": "object", "required": ["checks"], "properties": { @@ -222,7 +222,7 @@ "type": "string" }, "validation_score": { - "description": "Aggregate quality score from question validation (`questions generate --validate`), 0-100.", + "description": "Aggregate quality score from question validation (`eval questions generate --validate`), 0-100.", "type": "number" }, "validation_details": { @@ -230,11 +230,11 @@ "type": "object" }, "trigger_lint": { - "description": "`catalog trigger-set` lint of the query against the catalog (`names_skill`, `echo_terms`, `exam_framing`, `lexical_rank`, `unknown_expected`, `label_unresolved`, ...).", + "description": "`eval triggers generate` lint of the query against the catalog (`names_skill`, `echo_terms`, `exam_framing`, `lexical_rank`, `unknown_expected`, `label_unresolved`, ...).", "type": "object" }, "source_text": { - "description": "The question text before `catalog trigger-set --rewrite` turned it into the user-voice `question`; absent when it was not rewritten.", + "description": "The question text before `eval triggers generate --rewrite` turned it into the user-voice `question`; absent when it was not rewritten.", "type": "string" } } diff --git a/dokimi/dokimi/schemas/task_inspect.v1.json b/dokimi/dokimi/schemas/task_inspect.v1.json index aa3594e30..d3459a383 100644 --- a/dokimi/dokimi/schemas/task_inspect.v1.json +++ b/dokimi/dokimi/schemas/task_inspect.v1.json @@ -2,13 +2,13 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/intel/dokimi/schemas/task_inspect.v1.json", "title": "Eval-task format inspection v1", - "description": "What `tasks inspect ` found under a path, from the files alone: nothing is imported, run or converted (dokimi/docs/coding-harnesses.md). One item per task directory or dataset file a format probe claims; a path nothing claims is a single `unknown` item. `details` is per format and open; every other level is closed.", + "description": "What `eval inspect ` found under a path, from the files alone: nothing is imported, run or converted (dokimi/docs/coding-harnesses.md). One item per task directory or dataset file a format probe claims; a path nothing claims is a single `unknown` item. `details` is per format and open; every other level is closed.", "type": "object", "required": ["schema_version", "root", "items"], "additionalProperties": false, "properties": { "schema_version": {"const": "task_inspect.v1"}, - "root": {"type": "string", "description": "The path given to `tasks inspect`, as given."}, + "root": {"type": "string", "description": "The path given to `eval inspect`, as given."}, "items": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/item"}} }, "$defs": { @@ -41,7 +41,7 @@ "consumers": { "type": "array", "minItems": 1, - "items": {"enum": ["harbor", "docker-*", "benchflow", "questions track", "explore track", "catalog trigger-set", "questions import", "none-yet"]}, + "items": {"enum": ["harbor", "docker-*", "benchflow", "questions track", "explore track", "catalog trigger-set", "questions import", "eval triggers generate", "eval import", "none-yet"]}, "description": "What in dokimi reads the item today; `none-yet` when nothing does." }, "alternatives": { diff --git a/dokimi/dokimi/task_formats/__init__.py b/dokimi/dokimi/task_formats/__init__.py index 12f9ab34d..a636f311b 100644 --- a/dokimi/dokimi/task_formats/__init__.py +++ b/dokimi/dokimi/task_formats/__init__.py @@ -4,7 +4,7 @@ """Eval-task format detection: which format is a path, and what does it hold? Detection and a structure report only: nothing here runs or converts a task. -``dokimi tasks inspect `` is the command line; :func:`inspect_path` the API. +``dokimi eval inspect `` is the command line; :func:`inspect_path` the API. """ from dokimi.task_formats.detect import DIR_PROBES, FILE_PROBES, PROBES, detect, inspect_path, unknown diff --git a/dokimi/dokimi/task_formats/imports/__init__.py b/dokimi/dokimi/task_formats/imports/__init__.py index 82a6b5d39..d751e6773 100644 --- a/dokimi/dokimi/task_formats/imports/__init__.py +++ b/dokimi/dokimi/task_formats/imports/__init__.py @@ -4,10 +4,10 @@ """Import adapters: answer-only eval formats -> dokimi questions with ``grading`` checks. The canonical model is the questions.v2 record: ``question`` (the prompt, sent verbatim), -``expected_topics`` (reference answers / points, for the judge and ``eval checklist``) and +``expected_topics`` (reference answers / points, for the judge and ``eval score checklist``) and ``grading`` (format-neutral checks, :mod:`dokimi.eval.reference_checks`). Each format here is only a translation into it; answering and scoring are the answer track's own -(``answers generate``, ``arms run``, ``eval reference``). :func:`import_path` picks the adapter +(``answers generate``, ``arms run``, ``eval score reference``). :func:`import_path` picks the adapter from the format :func:`~dokimi.task_formats.detect` reports, or the one named. """ @@ -19,7 +19,7 @@ from dokimi.task_formats.imports import inspect_ai, openai_evals, promptfoo, skill_creator from dokimi.task_formats.imports.common import PROMPT_VERBATIM, Imported -#: Format name (as ``tasks inspect`` reports it) -> adapter taking ``(path, options)``. +#: Format name (as ``eval inspect`` reports it) -> adapter taking ``(path, options)``. ADAPTERS: dict[str, Callable[..., Imported]] = { inspect_ai.FORMAT_SAMPLES: lambda p, scorer=None, **_: inspect_ai.import_samples(p, scorer=scorer), inspect_ai.FORMAT_TASK: lambda p, task=None, **_: inspect_ai.import_task(p, task=task), @@ -40,7 +40,7 @@ def import_path(path: Path, *, fmt: Optional[str] = None, **options: Optional[st report = detect(Path(path)) if report is None: - raise ValueError(f"{path}: no known eval format detected (see `tasks inspect {path}`)") + raise ValueError(f"{path}: no known eval format detected (see `eval inspect {path}`)") if report.alternatives: # A file alone often fits two formats (Inspect samples vs OpenAI evals rows); only # the config that names it can tell, so a guess would import the wrong graders. diff --git a/dokimi/dokimi/task_formats/imports/common.py b/dokimi/dokimi/task_formats/imports/common.py index 1c6be2dfd..46f5d83bc 100644 --- a/dokimi/dokimi/task_formats/imports/common.py +++ b/dokimi/dokimi/task_formats/imports/common.py @@ -64,7 +64,7 @@ def question(qid: Any, text: str, grading: dict[str, Any], *, source_format: str "metadata": {"source_format": source_format, "source_path": str(path), **(metadata or {})}, } if refs: - # The stock judge and `eval checklist` read reference points from here. + # The stock judge and `eval score checklist` read reference points from here. record["expected_topics"] = refs return record diff --git a/dokimi/dokimi/task_formats/imports/native_logs.py b/dokimi/dokimi/task_formats/imports/native_logs.py index 64da32e5f..5ff0181fd 100644 --- a/dokimi/dokimi/task_formats/imports/native_logs.py +++ b/dokimi/dokimi/task_formats/imports/native_logs.py @@ -8,7 +8,7 @@ -- and read its log here. The completions become an ``answers.v1`` artifact (one condition, ``without_docs``: the native run had no dokimi doc context) and the tool's own verdicts an ``eval.v1`` artifact, so the native score sits next to dokimi's on the same records and -``eval reference`` can re-grade the same completions. Question ids follow the importers': the +``eval score reference`` can re-grade the same completions. Question ids follow the importers': the Inspect sample id, ```` / ``.p`` for promptfoo. Read for Inspect AI log format ``json`` (inspect-ai 0.3.283; ``samples[].output.completion``, @@ -56,7 +56,7 @@ def _add(run: NativeRun, qid: Any, text: str, completion: Optional[str], score: passed: Optional[bool], detail: Any = None, error: Optional[str] = None) -> None: answer = None if completion is None else {"answer": completion, "model": run.model} if answer is not None and error: - answer["error"] = error # a failed sample is counted under `errors` in `eval reference` + answer["error"] = error # a failed sample is counted under `errors` in `eval score reference` run.answers.append({"question_id": str(qid), "question_text": text, "without_docs": answer}) run.evaluations.append({"question_id": str(qid), "question_text": text, NATIVE_CONDITION: _block(score, passed, run.tool, detail)}) diff --git a/dokimi/dokimi/task_formats/imports/promptfoo.py b/dokimi/dokimi/task_formats/imports/promptfoo.py index 35f24f5a1..f4605e60c 100644 --- a/dokimi/dokimi/task_formats/imports/promptfoo.py +++ b/dokimi/dokimi/task_formats/imports/promptfoo.py @@ -44,7 +44,7 @@ def _text_check(base: str, value: Any, negate: bool) -> Optional[dict[str, Any]] """The deterministic check for a text assert, or None when ``base`` is not one.""" if base in _CONTAINS or base in ("equals", "starts-with", "regex"): # No value to compare (`regex` would search for "None"), a blank one, or an empty - # `contains-*` list, which eval reference refuses: left unsupported instead. + # `contains-*` list, which eval score reference refuses: left unsupported instead. if value is None or any(not str(v).strip() for v in as_list(value)): return None if base in _CONTAINS and not as_list(value): @@ -75,7 +75,7 @@ def _other_check(base: str, value: Any, negate: bool, imp: Imported) -> Optional return check if base in _JUDGED and not negate: if not str(value or "").strip(): - return None # a judge check with no criterion or reference is refused by eval reference + return None # a judge check with no criterion or reference is refused by eval score reference imp.note(f"{base}: graded by the yes/no checklist judge") return judge(native=base, **{_JUDGED[base]: str(value or "")}) return None diff --git a/dokimi/dokimi/task_formats/model.py b/dokimi/dokimi/task_formats/model.py index 6e278f478..74e2cb2a0 100644 --- a/dokimi/dokimi/task_formats/model.py +++ b/dokimi/dokimi/task_formats/model.py @@ -27,10 +27,10 @@ BY_BENCHFLOW = "benchflow" BY_QUESTIONS = "questions track" BY_EXPLORE = "explore track" -# Read back by `catalog trigger-set --trigger-set`, then run by the skill-catalog: arm. -BY_TRIGGER_SET = "catalog trigger-set" -BY_IMPORT = "questions import" -#: Formats ``questions import`` translates (the keys of ``task_formats.imports.ADAPTERS``). +# Read back by `eval triggers generate --trigger-set`, then run by the skill-catalog: arm. +BY_TRIGGER_SET = "eval triggers generate" +BY_IMPORT = "eval import" +#: Formats ``eval import`` translates (the keys of ``task_formats.imports.ADAPTERS``). IMPORTABLE = frozenset({"inspect-samples", "inspect-task", "openai-evals", "skill-creator-evals", "promptfoo"}) NONE_YET = "none-yet" @@ -117,7 +117,7 @@ def describe(self, source: Any) -> Report: ... def mark_importable(report: Report) -> Report: - """Name ``questions import`` as a consumer of a format it translates.""" + """Name ``eval import`` as a consumer of a format it translates.""" if report.format in IMPORTABLE and BY_IMPORT not in report.consumers: report.consumers.append(BY_IMPORT) return report diff --git a/dokimi/dokimi/treatments/arms.py b/dokimi/dokimi/treatments/arms.py index 16a774146..718445484 100644 --- a/dokimi/dokimi/treatments/arms.py +++ b/dokimi/dokimi/treatments/arms.py @@ -189,7 +189,7 @@ class SkillCatalogTreatment(Treatment): This is a tool-calling proxy, not a coding harness: a real harness renders the catalog as one listing it may truncate, and lets the model explore before it decides. Real-harness selection is measured with an external tool against the - trigger set ``catalog trigger-set`` writes (docs/evaluating-treatments.md). + trigger set ``eval triggers generate`` writes (docs/evaluating-treatments.md). """ arm_kind = "skill_catalog" diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index da8072efd..1bedc55af 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -57,15 +57,15 @@ dev = [ "types-PyYAML", "types-jsonschema", "scriv>=1.5", - # The test suite covers eval ragas. + # The test suite covers eval score ragas. "dokimi[ragas]", ] -# `eval ragas` and `answers generate --ragas`. +# `eval score ragas` and `answers generate --ragas`. ragas = [ "ragas>=0.3.0,<0.4", # Every ragas release imports langchain_community.chat_models.vertexai, which 0.4.2 removed. "langchain-community<0.4.2", - # eval ragas wraps the judge in ChatOpenAI or ChatAnthropic, typed as langchain-core's BaseChatModel. + # eval score ragas wraps the judge in ChatOpenAI or ChatAnthropic, typed as langchain-core's BaseChatModel. "langchain-anthropic>=1.0,<2", "langchain-openai>=1.0,<2", "langchain-core>=1.0,<2", @@ -73,9 +73,8 @@ ragas = [ ocr = ["docling>=2.0"] mcp = ["mcp>=1.0"] embeddings = ["sentence-transformers>=2.2"] -# Optional local baseline arm for `eval judge-validity --scorer typed:...`. -# Floor is the version the arm was measured with: the adapter relies on -# `predict(..., max_len=)` and on `usage.truncated` / `usage.state_tokens_dropped`. +# Optional baseline arm for `eval judge validate --scorer typed:...`; floor = measured version +# (the adapter needs `predict(max_len=)` and `usage.truncated` / `usage.state_tokens_dropped`). typed-scorer = ["laya>=0.3.24", "torch>=2.0"] [project.scripts] diff --git a/dokimi/tests/fixtures/task_formats/MANIFEST.yaml b/dokimi/tests/fixtures/task_formats/MANIFEST.yaml index 642888880..5da0db03b 100644 --- a/dokimi/tests/fixtures/task_formats/MANIFEST.yaml +++ b/dokimi/tests/fixtures/task_formats/MANIFEST.yaml @@ -1,4 +1,4 @@ -# What `dokimi tasks inspect` must report for each file of this corpus, and where +# What `dokimi eval inspect` must report for each file of this corpus, and where # each file came from. tests/test_task_format_corpus.py inspects the whole tree # (a mixed tree) once and checks every entry; `format: null` means the path must # not appear in the report. @@ -30,7 +30,7 @@ items: - path: catalog/skills/dpnp-io/evals/evals.json format: skill-creator-evals kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 3 source: intel-skills upstream: skills/dpnp-io/evals/evals.json @@ -71,14 +71,14 @@ items: - path: inspect/examples/hello_world.py format: inspect-task kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 1 source: inspect_ai upstream: examples/hello_world.py - path: inspect/examples/biology_qa.py format: inspect-task kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 1 source: inspect_ai upstream: examples/biology_qa.py @@ -86,7 +86,7 @@ items: - path: inspect/_examples/biology_qa.jsonl format: inspect-samples kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 20 source: inspect_ai upstream: src/inspect_ai/dataset/_examples/biology_qa.jsonl @@ -95,7 +95,7 @@ items: - path: inspect/_examples/security_guide.jsonl format: inspect-samples kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 5 source: inspect_ai upstream: src/inspect_ai/dataset/_examples/security_guide.jsonl @@ -103,14 +103,14 @@ items: - path: inspect/test_dataset/samples.csv format: inspect-samples kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 1 source: inspect_ai upstream: tests/dataset/test_dataset/samples.csv - path: inspect/test_dataset/samples-mcq.csv format: inspect-samples kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 1 source: inspect_ai upstream: tests/dataset/test_dataset/samples-mcq.csv @@ -157,7 +157,7 @@ items: - path: openai-evals/registry/data/allergen-information/samples.jsonl format: openai-evals kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 4 source: openai-evals upstream: evals/registry/data/allergen-information/samples.jsonl @@ -173,7 +173,7 @@ items: - path: openai-evals/registry/data/test_modelgraded/joke_fruits.jsonl format: openai-evals kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 10 source: openai-evals upstream: evals/registry/data/test_modelgraded/joke_fruits.jsonl @@ -184,7 +184,7 @@ items: - path: promptfoo/eval-custom-grader-csv/promptfooconfig.yaml format: promptfoo kind: answer-only - consumers: [questions import] + consumers: [eval import] items: null source: promptfoo upstream: examples/eval-custom-grader-csv/promptfooconfig.yaml @@ -202,7 +202,7 @@ items: - path: promptfoo/redteam-auth/promptfooconfig.file.yaml format: promptfoo kind: answer-only - consumers: [questions import] + consumers: [eval import] items: null source: promptfoo upstream: examples/redteam-auth/promptfooconfig.file.yaml @@ -212,28 +212,28 @@ items: - path: skill-creator/pdf-forms/evals/evals.json format: skill-creator-evals kind: answer-only - consumers: [questions import] + consumers: [eval import] items: 2 source: generated note: evals.json as anthropics/skills skills/skill-creator/references/schemas.md defines it - path: skill-reach/queries.json format: skill-reach kind: trigger - consumers: [catalog trigger-set] + consumers: [eval triggers generate] items: 3 source: generated note: QuerySet fields of google/skill-reach src/reach/queries.py and models.py (query_id, kind, acceptable_skills) - path: writers/pdf-forms.trigger.json format: skill-creator-trigger kind: trigger - consumers: [catalog trigger-set] + consumers: [eval triggers generate] items: 3 source: generated note: dokimi.external.trigger_set.to_skill_creator (tests/test_task_format_corpus.py rewrites it) - path: writers/reach-queries.json format: skill-reach kind: trigger - consumers: [catalog trigger-set] + consumers: [eval triggers generate] items: 3 source: generated note: dokimi.external.trigger_set.to_reach (legacy `id` spelling) diff --git a/dokimi/tests/outcomes/journeys.py b/dokimi/tests/outcomes/journeys.py index 8eb092b26..b20b417dc 100644 --- a/dokimi/tests/outcomes/journeys.py +++ b/dokimi/tests/outcomes/journeys.py @@ -180,7 +180,7 @@ def resolve(root: argparse.ArgumentParser, tokens: list[str]) -> tuple[argparse. def surface_problem(root: argparse.ArgumentParser, surface: str) -> str | None: - """Why ``surface`` (``"repo tasks check --warm"``) is not in the parser, or None.""" + """Why ``surface`` (``"eval check --warm"``) is not in the parser, or None.""" tokens = shlex.split(surface) if "--" in tokens: return f"a surface names options, not '--': {surface}" diff --git a/dokimi/tests/outcomes/journeys.yaml b/dokimi/tests/outcomes/journeys.yaml index 14ce6780c..5f3d993f7 100644 --- a/dokimi/tests/outcomes/journeys.yaml +++ b/dokimi/tests/outcomes/journeys.yaml @@ -260,12 +260,12 @@ criteria: delivery: available enforce: true capability: task-self-check - surfaces: ["repo tasks check --warm"] - refs: [docs/repo-evaluation-contract.md#43-self-check-tasks-check-no-llm, dokimi/commands/repo_run.py] + surfaces: ["eval check --warm"] + refs: [docs/repo-evaluation-contract.md#43-self-check-eval-check-no-llm, dokimi/commands/repo_run.py] evidence: - id: cjm-tasks-check-graders kind: cli_scenario - invokes: ["repo tasks check --manifest --out", "repo tasks check --warm"] + invokes: ["eval check --manifest --out", "eval check --warm"] stubs: [Docker (FakeRunner runs the task scripts on the host)] scope: verifier sanity on null and oracle; says nothing about task relevance or coverage (AC-J07-004) AC-J07-002: @@ -275,7 +275,7 @@ criteria: evidence: - id: cjm-tasks-check-graders kind: cli_scenario - invokes: ["repo tasks check --manifest --out"] + invokes: ["eval check --manifest --out"] stubs: [Docker (FakeRunner runs the task scripts on the host)] AC-J07-003: delivery: available @@ -284,11 +284,11 @@ criteria: evidence: - id: cjm-tasks-check-isolation kind: cli_scenario - invokes: ["repo tasks check --task"] + invokes: ["eval check --task"] stubs: [Docker (FakeRunner, with a simulated leaky image)] - id: cjm-tasks-check-host kind: cli_scenario - invokes: ["repo tasks check --warm", "repo tasks check --allow-unsupported"] + invokes: ["eval check --warm", "eval check --allow-unsupported"] stubs: [Docker (host probe fake)] gaps: ["isolation is checked by name and mount, against a simulated image, not a real container escape", "the probe's find root (/), its /proc /sys /dev prune and the source mount are not exercised: the shim searches a small tree"] @@ -299,13 +299,13 @@ criteria: evidence: - id: cjm-builtin-review-graders kind: cli_scenario - invokes: ["repo tasks check --task", "repo validate --json"] + invokes: ["eval check --task", "repo validate --json"] stubs: [Docker (FakeRunner)] AC-J07-005: delivery: partial enforce: false capability: judge-validity - surfaces: ["eval judge-validity --sequential"] + surfaces: ["eval judge validate --sequential"] # The test pins what happens today, which is not what the criterion asks for. evidence: - {id: cjm-judge-cap-inconclusive, kind: unit, stubs: [judge scores (fixed function)]} @@ -522,7 +522,7 @@ criteria: evidence: - id: cjm-tasks-check-host kind: cli_scenario - invokes: ["repo tasks check --allow-unsupported"] + invokes: ["eval check --allow-unsupported"] stubs: [Docker (host probe fake)] - {id: cjm-repo-eval-workflow, kind: unit, stubs: [dokimi (stub executable)]} gaps: diff --git a/dokimi/tests/test_answer_imports.py b/dokimi/tests/test_answer_imports.py index 37a1acadc..28904e4e2 100644 --- a/dokimi/tests/test_answer_imports.py +++ b/dokimi/tests/test_answer_imports.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for the answer-only format importers (``dokimi.task_formats.imports``) and -``questions import``. +``eval import``. Fixtures are hand-written from the upstream shapes the importers cite: @@ -291,7 +291,7 @@ def test_a_punctuation_only_ideal_is_a_real_check_not_a_pass(tmp_path, grader): def test_an_ambiguous_file_is_refused_unless_the_format_is_named(): - # Alone, joke_fruits.jsonl reads as Inspect samples or OpenAI evals rows (`tasks inspect` + # Alone, joke_fruits.jsonl reads as Inspect samples or OpenAI evals rows (`eval inspect` # lists the second as an alternative); only test-modelgraded.yaml says which it is. data = _EVALS / "data" / "test_modelgraded" / "joke_fruits.jsonl" with pytest.raises(ValueError, match=r"ambiguous format \(inspect-samples, openai-evals\).*--format"): @@ -535,7 +535,7 @@ def test_import_path_detects_the_format_and_refuses_one_without_an_adapter(tmp_p def test_questions_import_cli_writes_a_valid_questions_artifact(tmp_path, capsys): data = _jsonl(tmp_path / "born.jsonl", _BORN) out = tmp_path / "q.json" - args = build_parser().parse_args(["questions", "import", str(data), "--grader", "match", "-o", str(out)]) + args = build_parser().parse_args(["eval", "import", str(data), "--grader", "match", "-o", str(out)]) args.func(args) loaded = load_questions(out) assert loaded.source_version == "questions.v2" @@ -550,7 +550,7 @@ def test_eval_reference_cli_without_judge(tmp_path): a = tmp_path / "a.json" a.write_text(json.dumps({"answers": [{"question_id": "1", "question_text": "Q", "without_docs": {"answer": "yes"}}]})) out = tmp_path / "e.json" - args = build_parser().parse_args(["eval", "reference", "--answers", str(a), "--questions", str(q), + args = build_parser().parse_args(["eval", "score", "reference", "--answers", str(a), "--questions", str(q), "-o", str(out), "--no-judge"]) args.func(args) block = json.loads(out.read_text())["evaluations"][0]["without_docs"] @@ -565,7 +565,7 @@ def test_eval_reference_cli_without_judge_on_an_arms_run(tmp_path): a.write_text(json.dumps({"baseline_arm": "baseline", "arms": ["baseline"], "answers": [ {"question_id": "1", "question_text": "Q", "arms": {"baseline": {"answer": "Paris"}}}]})) out = tmp_path / "r.json" - args = build_parser().parse_args(["eval", "reference", "--answers", str(a), "--questions", str(q), + args = build_parser().parse_args(["eval", "score", "reference", "--answers", str(a), "--questions", str(q), "-o", str(out), "--no-judge"]) args.func(args) data = json.loads(out.read_text()) @@ -655,4 +655,11 @@ def test_importable_formats_match_the_adapters_and_show_as_consumer(tmp_path): assert IMPORTABLE == frozenset(ADAPTERS) report = detect(_jsonl(tmp_path / "samples.jsonl", _BORN)) - assert report.consumers == ["questions import"] + assert report.consumers == ["eval import"] + + +def test_import_path_without_a_format_names_eval_inspect_when_none_is_detected(tmp_path): + notes = tmp_path / "notes.txt" + notes.write_text("just some prose, no eval format here\n") + with pytest.raises(ValueError, match=r"see `eval inspect .*notes\.txt`"): + import_path(notes) diff --git a/dokimi/tests/test_bundled_skills.py b/dokimi/tests/test_bundled_skills.py index 60bfee2ce..fd96f71cc 100644 --- a/dokimi/tests/test_bundled_skills.py +++ b/dokimi/tests/test_bundled_skills.py @@ -25,7 +25,7 @@ SKILLS = Path(__file__).resolve().parents[1] / "data" / "skills" BUNDLED = sorted(d for d in SKILLS.iterdir() if (d / "SKILL.md").is_file()) LINK = re.compile(r"\]\(\s][^)#<>]*?)>?(?:#[^)\s]*)?(?:\s+\"[^\"]*\")?\)") -JUDGE_VALIDITY = re.compile(r"python cli\.py eval judge-validity[^`\n]*") +JUDGE_VALIDITY = re.compile(r"python cli\.py eval judge validate[^`\n]*") SHELL_BLOCK = re.compile(r"^```(?:bash|sh|shell)\s*\n(.*?)^```", re.M | re.S) diff --git a/dokimi/tests/test_catalog_command.py b/dokimi/tests/test_catalog_command.py index 9d0d79410..c67eb6df9 100644 --- a/dokimi/tests/test_catalog_command.py +++ b/dokimi/tests/test_catalog_command.py @@ -361,7 +361,7 @@ def test_catalog_trigger_set_reports_clean_error_on_malformed_skill_md(tmp_path, questions = tmp_path / "q.json" questions.write_text(json.dumps([{"id": "q1", "question": "hi"}]), encoding="utf-8") - args = build_parser().parse_args(["catalog", "trigger-set", "--catalog", str(catalog), + args = build_parser().parse_args(["eval", "triggers", "generate", "--catalog", str(catalog), "--questions", f"{questions}=none", "--out", str(tmp_path / "o.json")]) with pytest.raises(SystemExit) as exc_info: args.func(args) diff --git a/dokimi/tests/test_cli_eval_moves.py b/dokimi/tests/test_cli_eval_moves.py new file mode 100644 index 000000000..7da92def0 --- /dev/null +++ b/dokimi/tests/test_cli_eval_moves.py @@ -0,0 +1,155 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""#94 M1: the evaluation commands moved under ``eval``; each old path now tombstones. + +Every old path answers with ``moved: dokimi `` and exits 2, and never runs. Every new +path is a live leaf of the parser. +""" + +from __future__ import annotations + +import argparse +import shlex + +import pytest + +from dokimi import cli_routes +from dokimi.cli import build_parser, parse_args + +# (old argv, new argv): the 21 moves of #94 §4 M1, including #214's two (`questions import`, +# `eval reference`). +MOVES = [ + (("tasks", "inspect"), ("eval", "inspect")), + (("repo", "tasks", "check"), ("eval", "check")), + (("personas", "discover"), ("eval", "personas", "discover")), + (("personas", "approve"), ("eval", "personas", "validate")), + (("questions", "generate"), ("eval", "questions", "generate")), + (("questions", "analyze"), ("eval", "questions", "analyze")), + (("questions", "refine"), ("eval", "questions", "refine")), + (("questions", "panel-review"), ("eval", "questions", "review")), + (("questions", "import"), ("eval", "import")), + (("tasks", "list"), ("eval", "tasks", "list")), + (("catalog", "trigger-set"), ("eval", "triggers", "generate")), + (("eval", "score"), ("eval", "score", "pointwise")), + (("eval", "pairwise"), ("eval", "score", "pairwise")), + (("eval", "panel-score"), ("eval", "score", "panel")), + (("eval", "checklist"), ("eval", "score", "checklist")), + (("eval", "grounding"), ("eval", "score", "grounding")), + (("eval", "ragas"), ("eval", "score", "ragas")), + (("eval", "reference"), ("eval", "score", "reference")), + (("eval", "judge-validity"), ("eval", "judge", "validate")), + (("eval", "judge-wording"), ("eval", "judge", "wording")), + (("eval", "cross-judge"), ("eval", "judge", "agreement")), +] + + +# The retired roots: a bare root (or its help) names the new group. Not leaves, so not in MOVES. +ROOT_MOVES = [ + (("personas",), ("eval", "personas")), + (("questions",), ("eval", "questions")), +] + + +def _parser_at(path: tuple[str, ...]) -> argparse.ArgumentParser | None: + parser = build_parser() + for word in path: + sub = next((a for a in parser._actions if isinstance(a, argparse._SubParsersAction)), None) + if sub is None or word not in sub.choices: + return None + parser = sub.choices[word] + return parser + + +def _is_leaf(parser: argparse.ArgumentParser) -> bool: + return not any(isinstance(a, argparse._SubParsersAction) for a in parser._actions) + + +def test_the_tombstone_table_holds_exactly_the_moves(): + assert {(tomb.old, tomb.new) for tomb in cli_routes.TOMBSTONES} == set(MOVES) | set(ROOT_MOVES) + assert len(MOVES) == 21 + + +@pytest.mark.parametrize(("old", "new"), ROOT_MOVES, ids=[" ".join(old) for old, _ in ROOT_MOVES]) +@pytest.mark.parametrize("tail", [[], ["--help"], ["generate", "x.json"]], ids=["bare", "help", "leaf"]) +def test_a_retired_root_names_its_new_group(old, new, tail, capsys): + with pytest.raises(SystemExit) as exc: + parse_args([*old, *tail]) + assert exc.value.code == 2 + assert f"moved: dokimi {shlex.join(new)}" in capsys.readouterr().err + + +@pytest.mark.parametrize(("old", "new"), MOVES, ids=[" ".join(old) for old, _ in MOVES]) +def test_an_old_path_exits_2_and_names_the_new_argv(old, new, capsys): + # An option after the old path: a bare `eval score` is the group's help, and a plain word after + # it is a mistyped leaf (see the test below), so neither is an old call to name here. + with pytest.raises(SystemExit) as exc: + parse_args([*old, "--answers", "a.json"]) + assert exc.value.code == 2 + assert f"moved: dokimi {shlex.join(new)}" in capsys.readouterr().err + + +def test_a_typo_after_a_group_that_stays_is_not_an_old_call(capsys): + # `eval score` is still a group, so `eval score pointwse` is a mistyped leaf: argparse names the + # choices. Reporting it as a moved command would suggest a command that does not exist. + argv = ["eval", "score", "pointwse", "--answers", "a.json"] + assert cli_routes.moved_argv(argv) is None + with pytest.raises(SystemExit) as exc: + parse_args(argv) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "moved:" not in err + assert "invalid choice" in err + + +@pytest.mark.parametrize(("old", "new"), MOVES, ids=[" ".join(old) for old, _ in MOVES]) +def test_an_old_path_keeps_its_arguments_when_it_tombstones(old, new, capsys): + argv = [*old, "--answers", "a.json", "pos"] + with pytest.raises(SystemExit) as exc: + parse_args(argv) + assert exc.value.code == 2 + assert f"moved: dokimi {shlex.join([*new, '--answers', 'a.json', 'pos'])}" in capsys.readouterr().err + + +@pytest.mark.parametrize(("old", "new"), MOVES, ids=[" ".join(old) for old, _ in MOVES]) +def test_the_new_path_is_a_live_leaf(old, new): + parser = _parser_at(new) + assert parser is not None, shlex.join(new) + assert _is_leaf(parser), shlex.join(new) + assert cli_routes.moved_argv(list(new)) is None + + +def test_the_retired_roots_are_gone_from_the_parser(): + top = build_parser() + sub = next(a for a in top._actions if isinstance(a, argparse._SubParsersAction)) + assert "personas" not in sub.choices + assert "questions" not in sub.choices + # A bare `questions` is a tombstone, not a route: it names `eval questions` and exits 2. + with pytest.raises(SystemExit) as exc: + parse_args(["questions"]) + assert exc.value.code == 2 + + +@pytest.mark.parametrize("argv", [["eval", "score"], ["eval", "score", "--help"]]) +def test_the_group_that_stays_answers_its_own_help(argv): + # `eval score` is still a group: bare, or with only help, it is not an old call to move. + assert cli_routes.moved_argv(argv) is None + + +@pytest.mark.parametrize("argv", [["eval", "score", "--answers=a.json"], ["eval", "score", "-x"]]) +def test_an_old_call_with_only_flags_still_tombstones(argv): + # The equals form and any other option are old calls: they name the new command. + assert cli_routes.moved_argv(argv) == ["eval", "score", "pointwise", *argv[2:]] + + +def test_eval_score_help_lists_its_leaves(capsys): + with pytest.raises(SystemExit) as exc: + parse_args(["eval", "score", "--help"]) + assert exc.value.code == 0 + assert "pointwise" in capsys.readouterr().out + + +def test_eval_score_with_a_leaf_is_not_a_tombstone(): + # `eval score` moved to `eval score pointwise`, so it is a group: its leaves are live argv. + for leaf in ("pointwise", "pairwise", "panel", "checklist", "grounding", "ragas", "reference"): + assert cli_routes.moved_argv(["eval", "score", leaf, "--x"]) is None, leaf diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 3e217687c..401f42ad3 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -92,7 +92,7 @@ def test_help_lists_each_root_under_its_section(): loop, ), **dict.fromkeys(("run", "compare", "library", "catalog", "novelty", "repo"), subjects), - **dict.fromkeys(("personas", "questions", "eval"), evals), + **dict.fromkeys(("eval",), evals), **dict.fromkeys(("dataset", "ingest", "tools", "context", "runs", "backends", "data"), data), } @@ -109,7 +109,6 @@ def test_help_lists_each_root_under_its_section(): ("subjects", "run"), ("tasks", "run"), ("explore", "run"), - ("catalog", "trigger-set"), ("catalog", "trigger-probe"), ("novelty", "analyze"), ("novelty", "atoms"), @@ -118,21 +117,22 @@ def test_help_lists_each_root_under_its_section(): ("novelty", "routing"), ("repo", "reach"), ("repo", "run"), - ("personas", "discover"), - ("questions", "generate"), - ("questions", "analyze"), - ("questions", "refine"), - ("questions", "panel-review"), - ("eval", "score"), - ("eval", "ragas"), - ("eval", "judge-validity"), - ("eval", "checklist"), - ("eval", "judge-wording"), - ("eval", "cross-judge"), - ("eval", "pairwise"), - ("eval", "reference"), + ("eval", "personas", "discover"), + ("eval", "questions", "generate"), + ("eval", "questions", "analyze"), + ("eval", "questions", "refine"), + ("eval", "questions", "review"), + ("eval", "triggers", "generate"), + ("eval", "score", "pointwise"), + ("eval", "score", "pairwise"), + ("eval", "score", "panel"), + ("eval", "score", "checklist"), + ("eval", "score", "ragas"), + ("eval", "score", "reference"), + ("eval", "judge", "validate"), + ("eval", "judge", "wording"), + ("eval", "judge", "agreement"), ("eval", "redteam"), - ("eval", "panel-score"), ("tools", "preference"), } @@ -190,12 +190,12 @@ def test_a_root_costs_paid_when_any_of_its_leaves_is_paid(): assert cli_routes.cost_of(("repo",)) == "paid" assert cli_routes.cost_of(("library",)) == "free" assert cli_routes.cost_of(("data",)) == "free" - assert cli_routes.cost_of(("eval", "judge-validity")) == "paid" + assert cli_routes.cost_of(("eval", "judge")) == "paid" assert cli_routes.cost_of(("no", "such", "route")) is None -def test_the_tombstone_table_is_empty_and_its_check_finds_a_live_route(): - assert cli_routes.TOMBSTONES == () +def test_the_tombstone_table_is_clean_and_its_check_finds_a_live_route(): + assert cli_routes.TOMBSTONES assert cli_routes.tombstone_problems() == [] bad = (cli_routes.Tombstone(("eval", "score"), ("eval", "inspect")),) assert cli_routes.tombstone_problems(bad) # the old path is still a live route @@ -265,7 +265,7 @@ def test_every_alias_points_at_its_own_root_parser(): def test_a_tombstone_rewrites_the_argv_and_exits_2(monkeypatch, capsys): - # Test-only entry: no command moves in this release. + # Test-only entry: `old-tool` is not a real command; the table is replaced for this test. monkeypatch.setattr( cli_routes, "TOMBSTONES", @@ -288,7 +288,7 @@ def test_data_schema_show_prints_the_route_table_as_json(capsys): doc = json.loads(capsys.readouterr().out) assert doc == cli_routes.document() assert doc["schema_version"] == "cli_routes.v1" - assert {"path": ["eval", "score"], "cost": "paid"} in doc["leaves"] + assert {"path": ["eval", "score", "pointwise"], "cost": "paid"} in doc["leaves"] def test_the_route_document_reports_each_roots_section_aliases_and_cost(): diff --git a/dokimi/tests/test_cross_judge_judge_errors.py b/dokimi/tests/test_cross_judge_judge_errors.py index 7bcb6cd18..4e354d8b0 100644 --- a/dokimi/tests/test_cross_judge_judge_errors.py +++ b/dokimi/tests/test_cross_judge_judge_errors.py @@ -86,7 +86,7 @@ def test_a_measure_read_from_a_file_asked_no_judge(): def _cross_judge(tmp_path, monkeypatch, failing, *extra, data=None): - """``eval cross-judge --judges anthropic/m`` with a judge that raises on *failing* answers. + """``eval judge agreement --judges anthropic/m`` with a judge that raises on *failing* answers. Returns ``(exit code, stderr, the written JSON report)``; the code is 0 when it did not exit. """ @@ -103,7 +103,7 @@ def score_answer(self, *, answer, **_): answers = tmp_path / "run.json" answers.write_text(json.dumps(data or {"library_name": "L", "answers": _records()}), encoding="utf-8") out = tmp_path / "cj.json" - args = build_parser().parse_args(["eval", "cross-judge", "--answers", str(answers), "--judges", "anthropic/m", + args = build_parser().parse_args(["eval", "judge", "agreement", "--answers", str(answers), "--judges", "anthropic/m", "--output", str(out), *extra]) code = 0 try: diff --git a/dokimi/tests/test_empty_question_rule.py b/dokimi/tests/test_empty_question_rule.py index 5ce51ea49..e2c90822e 100644 --- a/dokimi/tests/test_empty_question_rule.py +++ b/dokimi/tests/test_empty_question_rule.py @@ -163,7 +163,7 @@ def test_cli_reports_the_record_and_exits_2(tmp_path, monkeypatch, capsys): answers = tmp_path / "answers.json" answers.write_text(json.dumps({"baseline_arm": "baseline", "answers": [_arms_record("q2", "")]}), encoding="utf-8") - monkeypatch.setattr(sys, "argv", ["dokimi", "eval", "pairwise", "--answers", str(answers)]) + monkeypatch.setattr(sys, "argv", ["dokimi", "eval", "score", "pairwise", "--answers", str(answers)]) monkeypatch.setattr("dokimi.commands.judge_reliability.pairwise_judge", lambda args: (MagicMock(), "stub/judge")) with pytest.raises(SystemExit) as exc: @@ -335,9 +335,9 @@ def evaluate(self, answers, include_without_docs=True): # ── no billed sampling probe before the refusal ────────────────────────── @pytest.mark.parametrize("cmd", [ - ["eval", "pairwise"], - ["eval", "checklist", "--questions", "{questions}"], - ["eval", "cross-judge", "--no-stored", "--judges", "anthropic/claude-haiku-4-5"], + ["eval", "score", "pairwise"], + ["eval", "score", "checklist", "--questions", "{questions}"], + ["eval", "judge", "agreement", "--no-stored", "--judges", "anthropic/claude-haiku-4-5"], ]) def test_no_probe_call_before_the_refusal(cmd, tmp_path, monkeypatch, capsys): from dokimi import cli diff --git a/dokimi/tests/test_eval_grounding_output_errors.py b/dokimi/tests/test_eval_grounding_output_errors.py index 2b00baf58..a22c701b1 100644 --- a/dokimi/tests/test_eval_grounding_output_errors.py +++ b/dokimi/tests/test_eval_grounding_output_errors.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""`eval grounding` and `eval judge-validity` exit 2 naming the path when --output cannot be written.""" +"""`eval score grounding` and `eval judge validate` exit 2 naming the path when --output cannot be written.""" from __future__ import annotations @@ -20,7 +20,7 @@ def _grounding(tmp_path: Path, output: Path) -> None: "with_docs": {"answer": "use parallel_for", "retrieved_docs": [{"snippet": "parallel_for runs tasks"}]}}, ]}), encoding="utf-8") - args = build_parser().parse_args(["eval", "grounding", "--answers", str(answers), "--output", str(output)]) + args = build_parser().parse_args(["eval", "score", "grounding", "--answers", str(answers), "--output", str(output)]) args.func(args) @@ -58,7 +58,7 @@ def test_judge_validity_output_below_a_file_exits_2_naming_the_path(tmp_path, mo blocker = tmp_path / "blocker" blocker.write_text("x", encoding="utf-8") target = blocker / "validity.json" - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), "--bases", "20", + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--bases", "20", "--variants", "truncate_20", "--output", str(target)]) with pytest.raises(SystemExit) as exc: args.func(args) diff --git a/dokimi/tests/test_eval_output_write_errors.py b/dokimi/tests/test_eval_output_write_errors.py index 0feb5e7ca..f165a69d1 100644 --- a/dokimi/tests/test_eval_output_write_errors.py +++ b/dokimi/tests/test_eval_output_write_errors.py @@ -82,10 +82,10 @@ def _stub_judge_commands(monkeypatch): _JUDGE_COMMANDS = { - "pairwise": ["eval", "pairwise"], - "checklist": ["eval", "checklist", "--questions", "q.json"], - "judge-wording": ["eval", "judge-wording"], - "cross-judge": ["eval", "cross-judge", "--judges", "openai/m", "--no-stored"], + "pairwise": ["eval", "score", "pairwise"], + "checklist": ["eval", "score", "checklist", "--questions", "q.json"], + "judge-wording": ["eval", "judge", "wording"], + "cross-judge": ["eval", "judge", "agreement", "--judges", "openai/m", "--no-stored"], } _WRITES_MD = ["pairwise", "checklist", "cross-judge"] @@ -164,7 +164,7 @@ def evaluate_answers(self, *a, **k): evaluate, "_load_answers", lambda path: types.SimpleNamespace(records=[], meta={}) ) args = build_parser().parse_args( - ["eval", "panel-score", "--product", "P", "--answers", "a.json", "--output", str(target)] + ["eval", "score", "panel", "--product", "P", "--answers", "a.json", "--output", str(target)] ) with pytest.raises(SystemExit) as exc: args.func(args) @@ -187,7 +187,7 @@ def save_evaluations(self, evals, path): monkeypatch.setattr(eval_mod, "Judge", lambda **kw: Judge()) target = tmp_path / "e.json" args = build_parser().parse_args( - ["eval", "score", "--product", "P", "--answers", str(answers), "--output", str(target)] + ["eval", "score", "pointwise", "--product", "P", "--answers", str(answers), "--output", str(target)] ) with pytest.raises(SystemExit) as exc: args.func(args) diff --git a/dokimi/tests/test_eval_questions_file_errors.py b/dokimi/tests/test_eval_questions_file_errors.py index 808295c25..acc648c62 100644 --- a/dokimi/tests/test_eval_questions_file_errors.py +++ b/dokimi/tests/test_eval_questions_file_errors.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""eval checklist / cross-judge --grounded: an unreadable --questions file is bad input (exit 2).""" +"""eval score checklist / cross-judge --grounded: an unreadable --questions file is bad input (exit 2).""" from __future__ import annotations @@ -18,8 +18,8 @@ def _arms_file(tmp_path): @pytest.mark.parametrize("command", [ - ["checklist"], - ["cross-judge", "--grounded", "--no-stored"], + ["score", "checklist"], + ["judge", "agreement", "--grounded", "--no-stored"], ], ids=["checklist", "cross-judge-grounded"]) @pytest.mark.parametrize("content", [None, "{not json", "\xff\xfe"], ids=["missing", "malformed", "not-utf8"]) diff --git a/dokimi/tests/test_eval_score_method_version.py b/dokimi/tests/test_eval_score_method_version.py index 9c8c30a7c..53d51df60 100644 --- a/dokimi/tests/test_eval_score_method_version.py +++ b/dokimi/tests/test_eval_score_method_version.py @@ -39,7 +39,7 @@ def _eval_score(tmp_path, answers_doc): answers = tmp_path / "answers.json" answers.write_text(json.dumps(answers_doc), encoding="utf-8") out = tmp_path / "eval.json" - args = build_parser().parse_args(["eval", "score", "--product", "L", "--answers", str(answers), + args = build_parser().parse_args(["eval", "score", "pointwise", "--product", "L", "--answers", str(answers), "--output", str(out), "--concurrency", "1", *JUDGE]) args.func(args) return json.loads(out.read_text(encoding="utf-8")) diff --git a/dokimi/tests/test_exit_codes_boundaries.py b/dokimi/tests/test_exit_codes_boundaries.py index 44f598a9f..9ea9abb47 100644 --- a/dokimi/tests/test_exit_codes_boundaries.py +++ b/dokimi/tests/test_exit_codes_boundaries.py @@ -554,7 +554,7 @@ def test_external_content_walkthrough_states_the_exit_of_a_preflight_without_key assert "missing OPENAI_API_KEY for provider openai" in capsys.readouterr().err -# ── catalog trigger-set: input refusals are 2; a rewrite that returns nothing is 1 ───────── +# ── eval triggers generate: input refusals are 2; a rewrite that returns nothing is 1 ───────── def _skill_root(tmp_path): root = tmp_path / "skills" @@ -578,7 +578,7 @@ def test_trigger_set_rewrite_returning_nothing_keeps_exit_1(tmp_path, monkeypatc spec = _trigger_questions(tmp_path, [ {"id": "1", "text": "perf shows low IPC", "metadata": {"skill_under_test": "linux-perf"}}, ]) - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--questions", spec, "--rewrite", "--model", "m", "--out", str(tmp_path / "o.json")) with pytest.raises(SystemExit) as exc: cli.main() @@ -595,7 +595,7 @@ def test_trigger_set_question_without_text_exits_2(text_fields, tmp_path, monkey spec = _trigger_questions(tmp_path, [ {"id": "1", **text_fields, "metadata": {"skill_under_test": "linux-perf"}}, ]) - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--questions", spec, "--out", str(tmp_path / "o.json")) with pytest.raises(SystemExit) as exc: cli.main() @@ -611,7 +611,7 @@ def test_trigger_set_no_text_refused_before_any_rewrite_call(tmp_path, monkeypat {"id": "1", "text": "perf shows low IPC", "metadata": {"skill_under_test": "linux-perf"}}, {"id": "2", "metadata": {"skill_under_test": "linux-perf"}}, ]) - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--questions", spec, "--rewrite", "--model", "m", "--out", str(tmp_path / "o.json")) with pytest.raises(SystemExit) as exc: cli.main() @@ -625,7 +625,7 @@ def test_trigger_set_file_without_labels_is_valid_and_its_questions_stay_unlabel # A --trigger-set file whose questions carry no expected_skill is valid: they are read unlabelled. path = tmp_path / "unlabelled.json" path.write_text(json.dumps({"questions": [{"id": "u1", "question": "How do I cook rice?"}]}), encoding="utf-8") - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--format", "questions", "--trigger-set", str(path), "--out", str(tmp_path / "o.json")) cli.main() assert (tmp_path / "o.json").exists() @@ -640,7 +640,7 @@ def test_trigger_set_file_without_labels_is_refused_where_it_would_write_an_empt # would be written out as an empty set with exit 0. path = tmp_path / "unlabelled.json" path.write_text(json.dumps({"questions": [{"id": "u1", "question": "How do I cook rice?"}]}), encoding="utf-8") - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--trigger-set", str(path), *fmt, "--out", str(tmp_path / "o.json")) with pytest.raises(SystemExit) as exc: cli.main() @@ -654,7 +654,7 @@ def test_trigger_set_of_negatives_only_is_still_written_as_reach(tmp_path, monke path = tmp_path / "negatives.json" path.write_text(json.dumps({"queries": [{"id": "n1", "text": "What is the weather?", "expected_skill": None}]}), encoding="utf-8") - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--trigger-set", str(path), "--out", str(tmp_path / "o.json")) cli.main() assert [q["id"] for q in json.loads((tmp_path / "o.json").read_text(encoding="utf-8"))["queries"]] == ["n1"] @@ -665,7 +665,7 @@ def test_questions_file_with_a_field_label_and_no_labels_is_refused_and_says_so( # negatives: exit 2, and the message says the file has no labels. path = tmp_path / "unlabelled.json" path.write_text(json.dumps({"questions": [{"id": "u1", "question": "How do I cook rice?"}]}), encoding="utf-8") - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--format", "questions", "--questions", f"{path}=@expected_skill", "--out", str(tmp_path / "o.json")) with pytest.raises(SystemExit) as exc: cli.main() @@ -682,7 +682,7 @@ def test_trigger_set_unresolved_field_rule_is_refused_before_any_rewrite_call(tm path.write_text(json.dumps({"questions": [ {"id": str(i), "text": f"question {i}", "metadata": {"skill_under_test": "linux-perf"}} for i in range(3) ]}), encoding="utf-8") - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--questions", f"{path}=@metadata.skil_under_test", "--rewrite", "--model", "m", "--out", str(tmp_path / "o.json")) with pytest.raises(SystemExit) as exc: @@ -700,7 +700,7 @@ def test_trigger_set_strict_refusal_exits_1(tmp_path, monkeypatch, capsys): {"id": "1", "text": "perf shows low IPC", "metadata": {"skill_under_test": "linux-perf"}}, {"id": "2", "text": "what is the weather"}, ]) - _argv(monkeypatch, "catalog", "trigger-set", "--catalog", str(_skill_root(tmp_path)), + _argv(monkeypatch, "eval", "triggers", "generate", "--catalog", str(_skill_root(tmp_path)), "--questions", spec, "--strict", "--out", str(tmp_path / "o.json")) with pytest.raises(SystemExit) as exc: cli.main() @@ -736,7 +736,7 @@ def similarity_failed(self, questions): raise ValueError("similarity backend returned NaN") monkeypatch.setattr(QuestionRefiner, "_deduplicate", similarity_failed) - _argv(monkeypatch, "questions", "refine", "--questions", _questions(tmp_path), + _argv(monkeypatch, "eval", "questions", "refine", "--questions", _questions(tmp_path), "--product", "numba", "--dry-run") with pytest.raises(ValueError, match="similarity backend returned NaN"): cli.main() diff --git a/dokimi/tests/test_exit_codes_input_errors.py b/dokimi/tests/test_exit_codes_input_errors.py index 7da5ad766..b798aab1f 100644 --- a/dokimi/tests/test_exit_codes_input_errors.py +++ b/dokimi/tests/test_exit_codes_input_errors.py @@ -21,11 +21,11 @@ def _missing(tmp_path): def _personas_approve(tmp_path): - return ["personas", "approve", "--file", _missing(tmp_path)] + return ["eval", "personas", "validate", "--file", _missing(tmp_path)] def _personas_discover_no_source(tmp_path): - return ["personas", "discover", "--product", "lib"] + return ["eval", "personas", "discover", "--product", "lib"] def _tools_preference_missing_groups(tmp_path): diff --git a/dokimi/tests/test_journeys_trace.py b/dokimi/tests/test_journeys_trace.py index 3dfc99a8f..e9ee0a46f 100644 --- a/dokimi/tests/test_journeys_trace.py +++ b/dokimi/tests/test_journeys_trace.py @@ -552,7 +552,7 @@ def test_a_short_alias_counts_as_its_long_option(): def test_an_abbreviation_argparse_accepts_counts_as_its_option(): - assert journeys.invoked(ROOT, "repo tasks check --task", [["repo", "tasks", "check", "src", "--ta", "x"]]) + assert journeys.invoked(ROOT, "eval check --task", [["eval", "check", "src", "--ta", "x"]]) def test_text_after_the_separator_is_not_an_option(): diff --git a/dokimi/tests/test_judge_not_measured.py b/dokimi/tests/test_judge_not_measured.py index d8ed359cc..9a5bad7f2 100644 --- a/dokimi/tests/test_judge_not_measured.py +++ b/dokimi/tests/test_judge_not_measured.py @@ -55,7 +55,7 @@ def score_answer(self, **_): 2, "Error: amazon-bedrock/m (aggregate) scored none of its 16 answers"), (["--answers", str(empty)], {}, 1, "✗ incomplete"), ): - code = _strict_exit(["eval", "cross-judge", *argv, "--output", str(tmp_path / "cj.json"), "--strict"]) + code = _strict_exit(["eval", "judge", "agreement", *argv, "--output", str(tmp_path / "cj.json"), "--strict"]) assert code == exit_code assert why in capsys.readouterr().err summary = json.loads((tmp_path / "cj.json").read_text())["summary"] @@ -83,7 +83,7 @@ def score_answer(self, *, answer, **_): "metadata": {"arm_kind": "skill"}} path = tmp_path / "answers.json" path.write_text(json.dumps(data), encoding="utf-8") - code = _strict_exit(["eval", "cross-judge", "--answers", str(path), "--no-stored", "--judges", + code = _strict_exit(["eval", "judge", "agreement", "--answers", str(path), "--no-stored", "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json"), "--strict"]) summary = json.loads((tmp_path / "cj.json").read_text())["summary"] assert summary["arms"]["skill"]["label"] != "NOT MEASURED" @@ -110,7 +110,7 @@ def rows(**kw): answers = tmp_path / "answers.json" answers.write_text(json.dumps(validity_answers()), encoding="utf-8") monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: report) - code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + code = _strict_exit(["eval", "judge", "wording", "--answers", str(answers), "--bases", "2", "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) out = capsys.readouterr() assert report.summary()["unmeasured_conditions"] == ["persona"] @@ -121,7 +121,7 @@ def rows(**kw): assert report.summary()["fragile_variants"] == [] assert all(not r["fragile"] for r in report.summary()["per_variant"].values()) # Report-only without --strict: the same run prints NOT MEASURED and exits 0. - code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + code = _strict_exit(["eval", "judge", "wording", "--answers", str(answers), "--bases", "2", "--judge-provider", "amazon-bedrock", "--judge-model", "m"]) assert code == 0 assert "NOT MEASURED" in capsys.readouterr().out @@ -146,7 +146,7 @@ def errored(n): for n in (0, 4): # no pair built; every call errored report = WordingReport(judge="fake", reports={c: errored(n) for c in ("original", "persona")}) monkeypatch.setattr(judge_wording, "run_wording", lambda *a, _r=report, **k: _r) - code = _strict_exit(["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + code = _strict_exit(["eval", "judge", "wording", "--answers", str(answers), "--bases", "2", "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--strict"]) assert code == 1 out = capsys.readouterr() @@ -166,7 +166,7 @@ def test_cross_judge_strict_exits_1_when_require_validated_leaves_no_voter(tmp_p path = tmp_path / "run.json" path.write_text(json.dumps(_stamped_run()), encoding="utf-8") - code = _strict_exit(["eval", "cross-judge", "--answers", str(path), "--require-validated", + code = _strict_exit(["eval", "judge", "agreement", "--answers", str(path), "--require-validated", "--output", str(tmp_path / "cj.json"), "--strict"]) assert code == 1 assert "1 of 1 arm(s) NOT MEASURED: skill" in capsys.readouterr().err @@ -188,7 +188,7 @@ def score_answer(self, *, answer, **_): # prefers the baseline, the stored judg monkeypatch.setattr(judge_mod, "Judge", OppositeJudge) path = tmp_path / "run.json" path.write_text(json.dumps(_material_run()), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(path), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(path), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] assert _strict_exit(argv) == 0 capsys.readouterr() @@ -216,7 +216,7 @@ def score_answer(self, *, answer, **_): # the baseline beats skill, and skill2 ev["scores"]["skill2"] = dict(ev["scores"]["skill"]) path = tmp_path / "run.json" path.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(path), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(path), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] assert _strict_exit(argv) == 0 capsys.readouterr() @@ -241,7 +241,7 @@ def score_answer(self, *, answer, **_): # prefers the skill, as the stored judg monkeypatch.setattr(judge_mod, "Judge", AgreeingJudge) path = tmp_path / "run.json" path.write_text(json.dumps(_material_run()), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(path), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(path), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] assert _strict_exit([*argv, "--strict"]) == 0 assert "incomplete" not in capsys.readouterr().err @@ -268,7 +268,7 @@ def rows(degraded): answers = tmp_path / "answers.json" answers.write_text(json.dumps(validity_answers()), encoding="utf-8") monkeypatch.setattr(judge_wording, "run_wording", lambda *a, **k: report) - argv = ["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + argv = ["eval", "judge", "wording", "--answers", str(answers), "--bases", "2", "--judge-provider", "amazon-bedrock", "--judge-model", "m"] assert _strict_exit(argv) == 0 assert "FRAGILE" in capsys.readouterr().out diff --git a/dokimi/tests/test_judge_placeholder_entry_points.py b/dokimi/tests/test_judge_placeholder_entry_points.py index 31f69d931..c96aed5ae 100644 --- a/dokimi/tests/test_judge_placeholder_entry_points.py +++ b/dokimi/tests/test_judge_placeholder_entry_points.py @@ -46,7 +46,7 @@ def _eval_score(tmp_path, *flag): answers = tmp_path / "answers.json" answers.write_text(json.dumps(ANSWERS), encoding="utf-8") out = tmp_path / "eval.json" - _run(["eval", "score", "--product", "L", "--answers", str(answers), "--output", str(out), + _run(["eval", "score", "pointwise", "--product", "L", "--answers", str(answers), "--output", str(out), "--concurrency", "1", *JUDGE, *flag]) return json.loads(out.read_text(encoding="utf-8")) diff --git a/dokimi/tests/test_judge_provider_choices.py b/dokimi/tests/test_judge_provider_choices.py index 4cb944527..196a1374a 100644 --- a/dokimi/tests/test_judge_provider_choices.py +++ b/dokimi/tests/test_judge_provider_choices.py @@ -39,8 +39,8 @@ def test_every_judge_provider_flag_offers_exactly_judge_providers(): actions = dict(_judge_provider_actions(build_parser())) for flag in ("benchmark run --judge-provider", "arms run --judge-provider", "arms matrix-run --judge-provider", "subjects run --judge-provider", - "evaluate --judge-provider", "eval score --judge-provider", - "eval judge-validity --escalate-provider"): + "evaluate --judge-provider", "eval score pointwise --judge-provider", + "eval judge validate --escalate-provider"): assert flag in actions assert NOT_THE_JUDGE_CLASS <= actions.keys() for flag in NOT_THE_JUDGE_CLASS: diff --git a/dokimi/tests/test_judge_provider_default.py b/dokimi/tests/test_judge_provider_default.py index e1d8fc4d4..547ffa134 100644 --- a/dokimi/tests/test_judge_provider_default.py +++ b/dokimi/tests/test_judge_provider_default.py @@ -47,9 +47,9 @@ def walk(parser: argparse.ArgumentParser, path: list[str]) -> None: def test_the_commands_that_take_a_judge_provider_are_all_covered(): assert set(COMMANDS) == { - "evaluate", "eval score", "eval judge-validity", "eval checklist", "eval judge-wording", - "eval pairwise", "benchmark run", "benchmark preflight", "benchmark batch", "arms run", - "arms matrix-run", "subjects run", "eval reference", + "evaluate", "eval score pointwise", "eval judge validate", "eval score checklist", + "eval judge wording", "eval score pairwise", "benchmark run", "benchmark preflight", + "benchmark batch", "arms run", "arms matrix-run", "subjects run", "eval score reference", } @@ -66,7 +66,7 @@ def test_every_judge_command_defaults_the_model_to_its_provider(command): ("openai", DEFAULT_GENERATOR_MODEL), ]) def test_a_provider_alone_takes_the_default_judge_that_runs_there(provider, model): - args = parse_args([*COMMANDS["eval score"], "--judge-provider", provider]) + args = parse_args([*COMMANDS["eval score pointwise"], "--judge-provider", provider]) assert (args.judge_provider, args.judge_model) == (provider, model) assert type(args.judge_model) is str @@ -88,7 +88,7 @@ def test_an_explicit_judge_model_wins_in_either_order(order): def test_an_explicit_default_name_is_kept_on_another_provider(): # Naming the default judge yourself is a choice, not a left-out flag. - args = parse_args([*COMMANDS["eval score"], "--judge-provider", "openai", + args = parse_args([*COMMANDS["eval score pointwise"], "--judge-provider", "openai", "--judge-model", DEFAULT_JUDGE_MODEL]) assert args.judge_model == DEFAULT_JUDGE_MODEL @@ -120,5 +120,5 @@ def test_the_legacy_cli_shim_exports_the_resolving_parse_args(): spec.loader.exec_module(shim) assert "parse_args" in shim.__all__ assert shim.parse_args is parse_args - args = shim.parse_args([*COMMANDS["eval score"], "--judge-provider", "openai"]) + args = shim.parse_args([*COMMANDS["eval score pointwise"], "--judge-provider", "openai"]) assert args.judge_model == DEFAULT_GENERATOR_MODEL diff --git a/dokimi/tests/test_judge_reliability.py b/dokimi/tests/test_judge_reliability.py index 880641742..076f7a577 100644 --- a/dokimi/tests/test_judge_reliability.py +++ b/dokimi/tests/test_judge_reliability.py @@ -419,7 +419,7 @@ def score_answer(self, *, answer, context="", **_): "skill": {"answer": "skill answer", "context_chunks": [{"snippet": "first"}, {"snippet": "second"}]}, }}]}), encoding="utf-8") - args = build_parser().parse_args(["eval", "cross-judge", "--answers", str(answers), "--no-stored", + args = build_parser().parse_args(["eval", "judge", "agreement", "--answers", str(answers), "--no-stored", "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")]) args.func(args) assert seen == {"skill answer": "first\nsecond", "base answer": "(No documentation retrieved)"} @@ -460,7 +460,7 @@ def _material_run(body=BODY, n=8): def _cross_judge(tmp_path, monkeypatch, data, *extra, judges=True): - """Run ``eval cross-judge`` on ``data`` with a recording fake Judge. + """Run ``eval judge agreement`` on ``data`` with a recording fake Judge. Returns ``(exit_code, contexts_seen, args)``; the code is 0 when it did not exit. """ @@ -482,7 +482,7 @@ def score_answer(self, **kw): monkeypatch.setattr(judge_mod, "Judge", FakeJudge) path = tmp_path / "run.json" path.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(path), "--output", str(tmp_path / "cj.json")] + argv = ["eval", "judge", "agreement", "--answers", str(path), "--output", str(tmp_path / "cj.json")] if judges: argv += ["--judges", "amazon-bedrock/m"] args = build_parser().parse_args(argv + list(extra)) @@ -604,7 +604,7 @@ def score_answer(self, **_): monkeypatch.setattr(judge_mod, "Judge", FakeJudge) answers = tmp_path / "run.json" answers.write_text(json.dumps(_stamped_run()), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -646,7 +646,7 @@ def score_answer(self, **kw): data["judge_materials"] = {digest: skill_body} answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) # Should not raise SystemExit; gate should pass because materials are intact. @@ -683,7 +683,7 @@ def score_answer(self, **_): # No judge_materials, but scores claim to be stored (no file before judge_context exists). answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -723,7 +723,7 @@ def score_answer(self, **kw): data["judge_materials"] = {digest: skill_body} answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) args.func(args) @@ -762,7 +762,7 @@ def score_answer(self, **_): pairwise.write_text(json.dumps({"verdicts": {"baseline": {}, "skill": {}}, "summary": {"judge": "p/m"}}), encoding="utf-8") answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--pairwise", str(pairwise), "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -797,7 +797,7 @@ def score_answer(self, **_): data["judge_materials"] = {digest: "ALTERED TEXT"} answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -835,7 +835,7 @@ def score_answer(self, **_): del ans["arms"]["skill"]["judge_context_sha256"] answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -871,7 +871,7 @@ def score_answer(self, **_): data["judge_materials"] = {digest1: skill_body} answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--judges", "amazon-bedrock/m", + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--judges", "amazon-bedrock/m", "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -900,7 +900,7 @@ def score_answer(self, **_): pairwise.write_text(json.dumps({"verdicts": {"baseline": {}, "skill": {}}, "summary": {"judge": "p/m"}}), encoding="utf-8") answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--pairwise", str(pairwise), + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--pairwise", str(pairwise), "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -929,7 +929,7 @@ def score_answer(self, **_): checklist.write_text(json.dumps({"questions": [], "summary": {"judge": "p/m"}}), encoding="utf-8") answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), "--checklist", str(checklist), + argv = ["eval", "judge", "agreement", "--answers", str(answers), "--checklist", str(checklist), "--output", str(tmp_path / "cj.json")] args = build_parser().parse_args(argv) with pytest.raises(SystemExit) as exc: @@ -973,7 +973,7 @@ def test_cross_judge_refuses_stored_scores_against_pairwise_and_checklist_too(tm other.write_text(json.dumps({"summary": {"judge": "p/j"}, "questions": [ {"question_id": q, "arms": {"baseline": {"coverage": 0.8}, "skill": {"coverage": 0.2}}} for q in qids]}), encoding="utf-8") - argv = ["eval", "cross-judge", "--answers", str(answers), measure_flag, str(other), + argv = ["eval", "judge", "agreement", "--answers", str(answers), measure_flag, str(other), "--output", str(tmp_path / "cj.json")] with pytest.raises(SystemExit) as exc: args = build_parser().parse_args(argv) @@ -1371,7 +1371,7 @@ def test_eval_pairwise_writes_the_errors_when_every_comparison_fails(tmp_path, m answers.write_text(json.dumps({"answers": [{"question_id": "Q1", "question_text": "q", "arms": { "baseline": {"answer": "b"}, "skill": {"answer": "s"}}}]}), encoding="utf-8") out = tmp_path / "pw.json" - args = build_parser().parse_args(["eval", "pairwise", "--answers", str(answers), "--output", str(out)]) + args = build_parser().parse_args(["eval", "score", "pairwise", "--answers", str(answers), "--output", str(out)]) with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 2 @@ -1443,7 +1443,7 @@ def test_a_judge_that_failed_its_gate_does_not_vote(): def test_escalation_without_pairwise_mode_is_refused(tmp_path): from dokimi.cli import build_parser - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(tmp_path / "a.json"), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(tmp_path / "a.json"), "--escalate-model", "m"]) with pytest.raises(SystemExit) as exc: args.func(args) @@ -1542,7 +1542,7 @@ def boom(self, *_, **__): answers.write_text(json.dumps({"answers": [{"question_id": "Q1", "question_text": "q", "arms": { "baseline": {"answer": "b"}, "skill": {"answer": "s"}}}]}), encoding="utf-8") questions.write_text(json.dumps({"questions": [{"id": "Q1", "expected_topics": ["p"]}]}), encoding="utf-8") - args = build_parser().parse_args(["eval", "checklist", "--answers", str(answers), + args = build_parser().parse_args(["eval", "score", "checklist", "--answers", str(answers), "--questions", str(questions), "--output", str(out)]) with pytest.raises(RuntimeError, match="render failed"): args.func(args) @@ -1618,7 +1618,7 @@ def _cross_judge_args(tmp_path, data, *extra): answers = tmp_path / "run.json" answers.write_text(json.dumps(data), encoding="utf-8") - return build_parser().parse_args(["eval", "cross-judge", "--answers", str(answers), + return build_parser().parse_args(["eval", "judge", "agreement", "--answers", str(answers), "--output", str(tmp_path / "cj.json"), *extra]) diff --git a/dokimi/tests/test_judge_sequential_design.py b/dokimi/tests/test_judge_sequential_design.py index 115b68e88..c4d326b5a 100644 --- a/dokimi/tests/test_judge_sequential_design.py +++ b/dokimi/tests/test_judge_sequential_design.py @@ -54,7 +54,7 @@ def test_the_command_exits_2_before_any_judge_call_for_an_undecidable_gate(tmp_p answers = tmp_path / "a.json" answers.write_text(json.dumps({"answers": []}), encoding="utf-8") args = build_parser().parse_args( - ["eval", "judge-validity", "--answers", str(answers), "--sequential", "--min-accuracy", "0.99"] + ["eval", "judge", "validate", "--answers", str(answers), "--sequential", "--min-accuracy", "0.99"] ) with pytest.raises(SystemExit) as exc: args.func(args) diff --git a/dokimi/tests/test_judge_temperature_entry_points.py b/dokimi/tests/test_judge_temperature_entry_points.py index a08e084a5..56082baea 100644 --- a/dokimi/tests/test_judge_temperature_entry_points.py +++ b/dokimi/tests/test_judge_temperature_entry_points.py @@ -55,7 +55,7 @@ def _exit_code(argv): @pytest.mark.parametrize("flag, expected", [(["--judge-temperature", "0"], 0.0), ([], None)]) def test_eval_score_forwards_and_records_judge_temperature(tmp_path, requests, flag, expected): out = tmp_path / "eval.json" - _run(["eval", "score", "--product", "L", "--answers", _answers(tmp_path), "--output", str(out), + _run(["eval", "score", "pointwise", "--product", "L", "--answers", _answers(tmp_path), "--output", str(out), "--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x", "--concurrency", "1", *flag]) assert len(requests) == 2 @@ -66,7 +66,7 @@ def test_eval_score_forwards_and_records_judge_temperature(tmp_path, requests, f def test_eval_panel_score_forwards_judge_temperature_to_every_judge(tmp_path, requests): out = tmp_path / "panel.json" - _run(["eval", "panel-score", "--product", "L", "--answers", _answers(tmp_path), + _run(["eval", "score", "panel", "--product", "L", "--answers", _answers(tmp_path), "--output", str(out), "--provider", "amazon-bedrock", "--model", "us.anthropic.x", "--roles", "technical_expert,doc_reviewer", "--concurrency", "1", "--judge-temperature", "0"]) @@ -77,7 +77,7 @@ def test_eval_panel_score_forwards_judge_temperature_to_every_judge(tmp_path, re def test_eval_panel_score_judge_configs_also_take_judge_temperature(tmp_path, requests): out = tmp_path / "panel.json" - _run(["eval", "panel-score", "--product", "L", "--answers", _answers(tmp_path), + _run(["eval", "score", "panel", "--product", "L", "--answers", _answers(tmp_path), "--output", str(out), "--judge-configs", "technical_expert:us.anthropic.x:amazon-bedrock", "--concurrency", "1", "--judge-temperature", "0"]) @@ -144,27 +144,27 @@ def _reliability_argv(command, tmp_path, out): arms.write_text(json.dumps(ARMS), encoding="utf-8") judge = ["--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x"] if command == "pairwise": - return ["eval", "pairwise", "--answers", str(arms), *judge, "--concurrency", "1", + return ["eval", "score", "pairwise", "--answers", str(arms), *judge, "--concurrency", "1", "--output", str(out)] if command == "checklist": questions = tmp_path / "q.json" questions.write_text(json.dumps({"questions": [{"id": f"Q{i}", "expected_topics": ["p"]} for i in range(3)]}), encoding="utf-8") - return ["eval", "checklist", "--answers", str(arms), "--questions", str(questions), *judge, + return ["eval", "score", "checklist", "--answers", str(arms), "--questions", str(questions), *judge, "--concurrency", "1", "--output", str(out)] if command == "judge-wording": answers = tmp_path / "answers.json" answers.write_text(json.dumps(validity_answers()), encoding="utf-8") - return ["eval", "judge-wording", "--answers", str(answers), "--bases", "2", + return ["eval", "judge", "wording", "--answers", str(answers), "--bases", "2", "--variants", "truncate_20", "--wordings", next(iter(WORDINGS)), "--redraws", "0", *judge, "--concurrency", "1", "--output", str(out)] if command == "judge-validity-pairwise": answers = tmp_path / "answers.json" answers.write_text(json.dumps(validity_answers()), encoding="utf-8") - return ["eval", "judge-validity", "--answers", str(answers), "--bases", "2", + return ["eval", "judge", "validate", "--answers", str(answers), "--bases", "2", "--variants", "truncate_20", "--mode", "pairwise", *judge, "--concurrency", "1", "--output", str(out)] - return ["eval", "cross-judge", "--answers", str(arms), "--no-stored", + return ["eval", "judge", "agreement", "--answers", str(arms), "--no-stored", "--judges", "amazon-bedrock/us.anthropic.x", "--concurrency", "1", "--output", str(out)] diff --git a/dokimi/tests/test_judge_validity.py b/dokimi/tests/test_judge_validity.py index 996219644..569d83564 100644 --- a/dokimi/tests/test_judge_validity.py +++ b/dokimi/tests/test_judge_validity.py @@ -7,7 +7,7 @@ really damage an answer, that they are reproducible, and that the accuracy and gate arithmetic is right for judges whose behaviour is known (a perfect judge, a saturated judge, an inverted judge). Measuring a real LLM judge needs API -credentials and is the job of `cli.py eval judge-validity`. +credentials and is the job of `cli.py eval judge validate`. """ import random @@ -523,7 +523,7 @@ def test_no_usable_answers_raises(self): def test_judge_validity_accepts_every_judge_provider(provider): from dokimi.cli import build_parser - args = build_parser().parse_args(["eval", "judge-validity", "--answers", "a.json", "--judge-provider", provider]) + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", "a.json", "--judge-provider", provider]) assert args.judge_provider == provider @@ -532,7 +532,7 @@ def test_judge_validity_rejects_providers_judge_rejects(provider): from dokimi.cli import build_parser with pytest.raises(SystemExit): - build_parser().parse_args(["eval", "judge-validity", "--answers", "a.json", "--judge-provider", provider]) + build_parser().parse_args(["eval", "judge", "validate", "--answers", "a.json", "--judge-provider", provider]) def test_noop_degradation_is_dropped_and_counted(): @@ -962,7 +962,7 @@ def _run(tmp_path, monkeypatch, capsys, bases, *extra, sign=1.0): ])) monkeypatch.setattr(judge_mod, "Judge", lambda **kw: None) monkeypatch.setattr(jv, "judge_scorer", lambda judge: lambda lib, q, a: sign * len(a)) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--bases", str(bases), "--variants", "truncate_20", *extra]) try: args.func(args) @@ -1046,7 +1046,7 @@ def test_report_is_written_before_a_console_that_cannot_print_it(tmp_path, monke monkeypatch.setattr(judge_mod, "Judge", lambda **kw: None) monkeypatch.setattr(jv, "judge_scorer", lambda judge: lambda lib, q, a: float(len(a))) monkeypatch.setattr(sys, "stdout", io.TextIOWrapper(io.BytesIO(), encoding="cp1252")) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--variants", "truncate_20", "--output", str(out)]) with pytest.raises(UnicodeEncodeError): args.func(args) @@ -1159,7 +1159,7 @@ def test_cli_warns_on_an_unmeasured_variant(tmp_path, monkeypatch, capsys, stric "arms": {"a": {"answer": f"Plain prose, answer {i}. " * 100}}} for i in range(20)])) monkeypatch.setattr(judge_mod, "Judge", lambda **kw: None) monkeypatch.setattr(jv, "judge_scorer", lambda judge: lambda lib, q, a: float(len(a))) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--variants", "strip_code,truncate_20", *strict]) try: args.func(args) @@ -1183,7 +1183,7 @@ def test_judge_temperature_reaches_the_judge_and_the_report(tmp_path, monkeypatc built = {} monkeypatch.setattr(judge_mod, "Judge", lambda **kw: built.update(kw)) monkeypatch.setattr(jv, "judge_scorer", lambda judge: lambda lib, q, a: float(len(a))) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--variants", "truncate_20", "--judge-temperature", "0", "--output", str(out)]) try: @@ -1589,7 +1589,7 @@ def test_cli_refuses_stored_context_it_cannot_honour(tmp_path, capsys, extra, me answers = tmp_path / "a.json" answers.write_text(json.dumps({"answers": _answers()})) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--context", "stored", *extra]) with pytest.raises(SystemExit) as exc: args.func(args) @@ -1616,7 +1616,7 @@ def score_answer(self, library_name, question, answer, context="(none passed)", answers = tmp_path / "arms.json" answers.write_text(json.dumps(_with_materials())) out = tmp_path / "v.json" - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--context", "stored", "--bases", "3", "--variants", "truncate_20", "--output", str(out)]) code = None @@ -1677,7 +1677,7 @@ def score_answer(self, library_name, question, answer, context="(none passed)", answers = tmp_path / "arms.json" answers.write_text(json.dumps(data if data is not None else _with_materials())) out = tmp_path / "v.json" - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--judge-provider", "amazon-bedrock", "--judge-model", "m", "--variants", "truncate_20", "--output", str(out), *argv]) code = None @@ -1706,7 +1706,7 @@ def test_cli_judge_validity_exits_2_when_no_answer_is_long_enough(tmp_path, monk answers = tmp_path / "a.json" answers.write_text(json.dumps({"answers": [{"question_id": "Q1", "question_text": "q", "arms": {"baseline": {"answer": "short"}}}]})) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--judge-provider", "amazon-bedrock"]) with pytest.raises(SystemExit) as exc: args.func(args) @@ -1768,7 +1768,7 @@ def gates(report_path): run_path = tmp_path / "run.json" run_path.write_text(json.dumps(run)) out = tmp_path / "cj.json" - args = build_parser().parse_args(["eval", "cross-judge", "--answers", str(run_path), "--no-stored", + args = build_parser().parse_args(["eval", "judge", "agreement", "--answers", str(run_path), "--no-stored", "--judges", "amazon-bedrock/m", "--validity", str(report_path), "--output", str(out)]) args.func(args) @@ -1961,7 +1961,7 @@ def test_cli_stored_context_refusal_names_the_real_cause(tmp_path, capsys, extra answers = tmp_path / "a.json" answers.write_text(json.dumps(_with_materials())) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--context", "stored", *extra]) with pytest.raises(SystemExit) as exc: args.func(args) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 8986ff57f..183ba0d89 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -258,7 +258,7 @@ def test_tools_or_a_conversation_are_refused_before_any_call(runner): for sampling in ({"temperature": 0.0}, {"seed": 7}): # --judge-temperature / --seed: before the run with pytest.raises(ValueError, match="temperature or seed"): Judge(model="sonnet", provider="claude-cli", **sampling) - with pytest.raises(ValueError, match="temperature or seed"): # eval panel-score + with pytest.raises(ValueError, match="temperature or seed"): # eval score panel JudgePanel(default_provider="claude-cli", **sampling) JudgePanel(default_provider="claude-cli") with pytest.raises(llm_cli.CliCallError, match="temperature"): @@ -434,8 +434,8 @@ def test_the_preflight_probes_a_cli_provider_once_for_its_error_and_its_warning( @pytest.mark.parametrize("argv", [ - ["eval", "score", "--judge-provider", "claude-cli", "--judge-temperature", "0"], - ["eval", "panel-score", "--provider", "claude-cli", "--judge-temperature", "0"], + ["eval", "score", "pointwise", "--judge-provider", "claude-cli", "--judge-temperature", "0"], + ["eval", "score", "panel", "--provider", "claude-cli", "--judge-temperature", "0"], ]) def test_a_cli_judge_given_a_temperature_is_an_error_line_and_exit_2(argv, tmp_path, monkeypatch, capsys): from dokimi import cli @@ -653,10 +653,10 @@ def test_every_command_with_a_hidden_or_default_model_refuses_the_cli_without_a_ cases = [ ["answers", "generate", "--product", "p", "--questions", "q.json", "--provider", "claude-cli"], - ["questions", "generate", "--product", "p", "--personas", "p.json", "--provider", "claude-cli"], - ["questions", "analyze", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], - ["questions", "panel-review", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], - ["personas", "discover", "--product", "p", "--provider", "claude-cli"], + ["eval", "questions", "generate", "--product", "p", "--personas", "p.json", "--provider", "claude-cli"], + ["eval", "questions", "analyze", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], + ["eval", "questions", "review", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], + ["eval", "personas", "discover", "--product", "p", "--provider", "claude-cli"], ["subjects", "run", "subject.toml", "--provider", "claude-cli"], ] for argv in cases: diff --git a/dokimi/tests/test_model_defaults.py b/dokimi/tests/test_model_defaults.py index 766b7b512..854c33f02 100644 --- a/dokimi/tests/test_model_defaults.py +++ b/dokimi/tests/test_model_defaults.py @@ -58,16 +58,16 @@ def test_no_source_file_still_names_gpt_4o_mini(): (["subjects", "run", "s.toml"], {"model": ANSWER, "judge": JUDGE}), (["evaluate", "--product", "p"], {"model": ANSWER, "judge": JUDGE}), (["benchmark", "run", "--target", "t"], {"model": ANSWER, "judge": JUDGE}), - (["eval", "score", "--answers", "a", "--product", "p"], {"judge": JUDGE}), - (["eval", "panel-score", "--answers", "a", "--product", "p"], {"model": JUDGE}), - (["eval", "ragas", "--answers", "a"], {"ragas": JUDGE}), + (["eval", "score", "pointwise", "--answers", "a", "--product", "p"], {"judge": JUDGE}), + (["eval", "score", "panel", "--answers", "a", "--product", "p"], {"model": JUDGE}), + (["eval", "score", "ragas", "--answers", "a"], {"ragas": JUDGE}), (["answers", "generate", "--product", "p", "--questions", "q"], {"model": ANSWER, "ragas": JUDGE}), - (["questions", "generate", "--product", "p", "--personas", "x"], {"model": GENERATOR}), - (["questions", "analyze", "--questions", "q", "--product", "p"], {"model": GENERATOR}), - (["questions", "refine", "--questions", "q", "--product", "p"], {"fill": GENERATOR}), - (["questions", "panel-review", "--questions", "q", "--product", "p"], {"model": GENERATOR}), - (["personas", "discover", "--product", "p"], {"model": GENERATOR}), + (["eval", "questions", "generate", "--product", "p", "--personas", "x"], {"model": GENERATOR}), + (["eval", "questions", "analyze", "--questions", "q", "--product", "p"], {"model": GENERATOR}), + (["eval", "questions", "refine", "--questions", "q", "--product", "p"], {"fill": GENERATOR}), + (["eval", "questions", "review", "--questions", "q", "--product", "p"], {"model": GENERATOR}), + (["eval", "personas", "discover", "--product", "p"], {"model": GENERATOR}), (["novelty", "analyze", "--artifact", "skill:x"], {"model": ANSWER}), (["novelty", "probe", "--artifact", "skill:x", "--atoms", "a"], {"model": ANSWER}), (["novelty", "atoms", "--artifact", "skill:x"], {"model": JUDGE}), diff --git a/dokimi/tests/test_openai_compatible_route.py b/dokimi/tests/test_openai_compatible_route.py index a0d283e33..9a02874c0 100644 --- a/dokimi/tests/test_openai_compatible_route.py +++ b/dokimi/tests/test_openai_compatible_route.py @@ -81,7 +81,7 @@ def test_cross_judge_scores_with_an_openai_compatible_judge(litellm_routes, tmp_ "arms": {"baseline": {"answer": f"b{i}"}, "skill": {"answer": f"s{i}"}}} for i in range(3)]}), encoding="utf-8") out = tmp_path / "cj.json" - args = build_parser().parse_args(["eval", "cross-judge", "--answers", str(answers), "--no-stored", + args = build_parser().parse_args(["eval", "judge", "agreement", "--answers", str(answers), "--no-stored", "--judges", "openai/glm-5.3", "--output", str(out)]) args.func(args) assert litellm_routes == ["openai/glm-5.3"] * 6 # three questions, two arms diff --git a/dokimi/tests/test_panel_properties.py b/dokimi/tests/test_panel_properties.py index 934ca5adb..9047d2797 100644 --- a/dokimi/tests/test_panel_properties.py +++ b/dokimi/tests/test_panel_properties.py @@ -145,14 +145,14 @@ def test_non_numeric_dimension_is_retried(raw): def test_cli_panel_score_end_to_end(tmp_path, monkeypatch, capsys, cache_dir): - """`dokimi eval panel-score` reads answers.v1 data and writes the panel file.""" + """`dokimi eval score panel` reads answers.v1 data and writes the panel file.""" from dokimi.cli import main answers = tmp_path / "a.json" answers.write_text(json.dumps({"answers": [_record("snippet", "question_text")]})) out = tmp_path / "out.json" comp, calls = _litellm([_reply()]) monkeypatch.setattr("litellm.completion", comp) - monkeypatch.setattr(sys, "argv", ["dokimi", "eval", "panel-score", "--answers", str(answers), + monkeypatch.setattr(sys, "argv", ["dokimi", "eval", "score", "panel", "--answers", str(answers), "--product", "lib", "--output", str(out)]) main() data = json.loads(out.read_text()) diff --git a/dokimi/tests/test_personas_approve.py b/dokimi/tests/test_personas_approve.py index bef1a5c27..3fcb20655 100644 --- a/dokimi/tests/test_personas_approve.py +++ b/dokimi/tests/test_personas_approve.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""`personas approve` rejects a persona file it cannot validate with a message, not a traceback.""" +"""`eval personas validate` rejects a persona file it cannot validate with a message, not a traceback.""" from __future__ import annotations @@ -15,7 +15,7 @@ def _approve(path, monkeypatch): - monkeypatch.setattr("sys.argv", ["dokimi", "personas", "approve", "--file", str(path)]) + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "personas", "validate", "--file", str(path)]) with pytest.raises(SystemExit) as exc: cli.main() return exc.value.code @@ -24,7 +24,7 @@ def _approve(path, monkeypatch): def test_approve_accepts_a_valid_persona_file(tmp_path, monkeypatch, capsys): f = tmp_path / "personas.json" f.write_text(json.dumps({"product": "oneTBB", "personas": [_PERSONA]}), encoding="utf-8") - monkeypatch.setattr("sys.argv", ["dokimi", "personas", "approve", "--file", str(f)]) + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "personas", "validate", "--file", str(f)]) cli.main() assert "Validated 1 personas" in capsys.readouterr().out diff --git a/dokimi/tests/test_personas_discover_output.py b/dokimi/tests/test_personas_discover_output.py index 3248088ef..5ef8fff6c 100644 --- a/dokimi/tests/test_personas_discover_output.py +++ b/dokimi/tests/test_personas_discover_output.py @@ -105,7 +105,7 @@ def save_report(self, report, path): qfile = tmp_path / "q.json" qfile.write_text('["a?"]', encoding="utf-8") out = _outputs(tmp_path)[kind] - monkeypatch.setattr("sys.argv", ["dokimi", "questions", "analyze", "--questions", str(qfile), + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "questions", "analyze", "--questions", str(qfile), "--product", "p", "--output", str(out)]) with pytest.raises(SystemExit) as exc: cli.main() diff --git a/dokimi/tests/test_printed_hints_parse.py b/dokimi/tests/test_printed_hints_parse.py index 7fff986a2..80680910f 100644 --- a/dokimi/tests/test_printed_hints_parse.py +++ b/dokimi/tests/test_printed_hints_parse.py @@ -56,7 +56,7 @@ def save_personas(self, personas, output_path): def test_personas_discover_next_step_parses(tmp_path, monkeypatch, capsys): monkeypatch.setattr(personas_cmd, "PersonaGenerator", _FakeGenerator) - args = cli.parse_args(["personas", "discover", "--product", "P", "--description", "d", + args = cli.parse_args(["eval", "personas", "discover", "--product", "P", "--description", "d", "--output", str(tmp_path / "P.json")]) args.func(args) _assert_hints_parse(capsys.readouterr().out) @@ -66,7 +66,7 @@ def test_personas_approve_next_step_parses(tmp_path, capsys): persona_file = tmp_path / "P.json" persona_file.write_text(json.dumps({"product": "P", "personas": [ {"id": "p1", "name": "n", "skill_level": "novice"}]}), encoding="utf-8") - args = cli.parse_args(["personas", "approve", "--file", str(persona_file)]) + args = cli.parse_args(["eval", "personas", "validate", "--file", str(persona_file)]) args.func(args) _assert_hints_parse(capsys.readouterr().out) diff --git a/dokimi/tests/test_provider_call_budgets.py b/dokimi/tests/test_provider_call_budgets.py index 47229e1e5..248928939 100644 --- a/dokimi/tests/test_provider_call_budgets.py +++ b/dokimi/tests/test_provider_call_budgets.py @@ -283,8 +283,7 @@ def test_eval_score_of_2_records_makes_4_provider_calls(provider_calls, tmp_path out = tmp_path / "eval.json" _run( [ - "eval", - "score", + "eval", "score", "pointwise", "--product", "lib", "--answers", @@ -307,8 +306,7 @@ def test_eval_panel_score_of_1_record_with_3_judges_makes_6_provider_calls( out = tmp_path / "panel.json" _run( [ - "eval", - "panel-score", + "eval", "score", "panel", "--product", "lib", "--answers", diff --git a/dokimi/tests/test_questions_artifact.py b/dokimi/tests/test_questions_artifact.py index 5ac6c7ebd..26e85e854 100644 --- a/dokimi/tests/test_questions_artifact.py +++ b/dokimi/tests/test_questions_artifact.py @@ -198,7 +198,7 @@ def test_questions_refine_writes_only_declared_keys(tmp_path): "difficulty": "hard", "persona": "dev"}, ]}), encoding="utf-8") out = tmp_path / "refined.json" - args = build_parser().parse_args(["questions", "refine", "--questions", str(src), + args = build_parser().parse_args(["eval", "questions", "refine", "--questions", str(src), "--product", "oneTBB", "--output", str(out)]) args.func(args) data = json.loads(out.read_text(encoding="utf-8")) @@ -217,7 +217,7 @@ def test_catalog_trigger_set_questions_writes_only_declared_keys(tmp_path): other.write_text(json.dumps([{"id": "n1", "text": "my cat is sad today"}]), encoding="utf-8") out = tmp_path / "trigger_questions.json" args = build_parser().parse_args([ - "catalog", "trigger-set", "--catalog", str(tmp_path / "skills"), "--format", "questions", + "eval", "triggers", "generate", "--catalog", str(tmp_path / "skills"), "--format", "questions", "--questions", f"{GOLDEN / 'onetbb_golden.excerpt.json'}=onetbb-quickstart", "--questions", f"{other}=none", "--rewrite", "--model", "m", "--out", str(out)]) with patch("dokimi.llm.llm_call", return_value="how do I speed up my loop"): @@ -636,7 +636,7 @@ def test_questions_refine_refuses_malformed_questions_with_error_exit(tmp_path, # Pass an empty dict where a list or questions object is expected: invalid structure src.write_text(json.dumps({}), encoding="utf-8") out = tmp_path / "refined.json" - args = build_parser().parse_args(["questions", "refine", "--product", "oneTBB", + args = build_parser().parse_args(["eval", "questions", "refine", "--product", "oneTBB", "--questions", str(src), "--output", str(out)]) with pytest.raises(SystemExit) as exc_info: args.func(args) @@ -650,7 +650,7 @@ def test_questions_refine_names_an_unreadable_questions_file(tmp_path, capsys, k src = tmp_path / "q.json" if kind == "directory": src.mkdir() - args = build_parser().parse_args(["questions", "refine", "--product", "oneTBB", + args = build_parser().parse_args(["eval", "questions", "refine", "--product", "oneTBB", "--questions", str(src), "--output", str(tmp_path / "out.json")]) with pytest.raises(SystemExit) as exc_info: args.func(args) @@ -746,7 +746,7 @@ def test_questions_refine_repairs_a_set_that_strict_reading_refuses(tmp_path, mo with pytest.raises(ArtifactValidationError, match="topics"): load_questions(src) out = tmp_path / "refined.json" - args = build_parser().parse_args(["questions", "refine", "--product", "oneTBB", + args = build_parser().parse_args(["eval", "questions", "refine", "--product", "oneTBB", "--questions", str(src), "--output", str(out)]) args.func(args) data = json.loads(out.read_text(encoding="utf-8")) @@ -761,7 +761,7 @@ def test_questions_refine_prints_an_error_when_the_refined_set_is_invalid(tmp_pa {"id": "q2", "text": "Why does my task_arena deadlock under nested parallelism?"}, ]}), encoding="utf-8") out = tmp_path / "refined.json" - args = build_parser().parse_args(["questions", "refine", "--product", "oneTBB", + args = build_parser().parse_args(["eval", "questions", "refine", "--product", "oneTBB", "--questions", str(src), "--output", str(out)]) with pytest.raises(SystemExit) as exc: args.func(args) @@ -785,7 +785,7 @@ def fail_fill(self, *a, **k): {"id": "q2", "text": "Why does my task_arena deadlock under nested parallelism?"}, ]}), encoding="utf-8") out = tmp_path / "refined.json" - args = build_parser().parse_args(["questions", "refine", "--product", "oneTBB", + args = build_parser().parse_args(["eval", "questions", "refine", "--product", "oneTBB", "--questions", str(src), "--output", str(out), "--fill-gaps"]) with pytest.raises(SystemExit) as exc: diff --git a/dokimi/tests/test_questions_generate_config.py b/dokimi/tests/test_questions_generate_config.py index 6cf77ea2f..e4c12d660 100644 --- a/dokimi/tests/test_questions_generate_config.py +++ b/dokimi/tests/test_questions_generate_config.py @@ -17,7 +17,7 @@ def _generate(tmp_path, monkeypatch, config_text): personas = tmp_path / "personas.json" personas.write_text('{"personas": []}', encoding="utf-8") args = build_parser().parse_args([ - "questions", "generate", "--product", "p", "--personas", str(personas), + "eval", "questions", "generate", "--product", "p", "--personas", str(personas), "--output", str(tmp_path / "q.json"), ]) args.func(args) diff --git a/dokimi/tests/test_questions_generate_inputs.py b/dokimi/tests/test_questions_generate_inputs.py index a874bfa0e..ad1cb3672 100644 --- a/dokimi/tests/test_questions_generate_inputs.py +++ b/dokimi/tests/test_questions_generate_inputs.py @@ -13,7 +13,7 @@ def _generate(monkeypatch, *extra): - monkeypatch.setattr("sys.argv", ["dokimi", "questions", "generate", "--product", "p", *extra]) + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "questions", "generate", "--product", "p", *extra]) with pytest.raises(SystemExit) as exc: cli.main() return exc.value.code @@ -112,7 +112,7 @@ def test_generate_completes_when_a_config_section_is_empty(tmp_path, monkeypatch topics.write_text("[]", encoding="utf-8") out = tmp_path / "q.json" monkeypatch.setattr("dokimi.questions.QuestionGenerator", _FakeGenerator) - monkeypatch.setattr("sys.argv", ["dokimi", "questions", "generate", "--product", "p", + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "questions", "generate", "--product", "p", "--personas", str(personas), "--topics", str(topics), "--output", str(out)]) cli.main() diff --git a/dokimi/tests/test_questions_output_unwritable.py b/dokimi/tests/test_questions_output_unwritable.py index 9daee81a9..74d15ae37 100644 --- a/dokimi/tests/test_questions_output_unwritable.py +++ b/dokimi/tests/test_questions_output_unwritable.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""questions refine and panel-review: an unwritable --output is a clean exit, not a traceback.""" +"""questions refine and review: an unwritable --output is a clean exit, not a traceback.""" from __future__ import annotations @@ -52,7 +52,7 @@ def out(tmp_path, request): def _run(monkeypatch, *argv): - monkeypatch.setattr("sys.argv", ["dokimi", "questions", *argv]) + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "questions", *argv]) with pytest.raises(SystemExit) as exc: cli.main() return exc.value.code @@ -71,7 +71,7 @@ def test_refine_exits_2_when_the_output_cannot_be_written(monkeypatch, capsys, q @pytest.mark.parametrize("out", ["under-a-file", "is-a-directory"], indirect=True) def test_panel_review_exits_2_when_the_output_cannot_be_written(monkeypatch, capsys, questions_file, out): monkeypatch.setattr("dokimi.questions.panel_reviewer.QuestionPanelReviewer", _FakeReviewer) - assert _run(monkeypatch, "panel-review", "--product", "p", "--questions", str(questions_file), + assert _run(monkeypatch, "review", "--product", "p", "--questions", str(questions_file), "--output", str(out)) == 2 err = capsys.readouterr().err assert err.startswith("Error: cannot write ") and str(out) in err diff --git a/dokimi/tests/test_ragas_runtime.py b/dokimi/tests/test_ragas_runtime.py index be8d9dde5..6f2730f65 100644 --- a/dokimi/tests/test_ragas_runtime.py +++ b/dokimi/tests/test_ragas_runtime.py @@ -7,7 +7,7 @@ ``langchain_*`` modules into ``sys.modules``, so the real import runs in a fresh interpreter. Every ragas release imports ``langchain_community.chat_models.vertexai``; langchain-community 0.4.2 removed -it, and nothing else noticed that ``eval ragas`` could not start. +it, and nothing else noticed that ``eval score ragas`` could not start. """ from __future__ import annotations diff --git a/dokimi/tests/test_record_readers.py b/dokimi/tests/test_record_readers.py index ee83a7108..752566fb1 100644 --- a/dokimi/tests/test_record_readers.py +++ b/dokimi/tests/test_record_readers.py @@ -443,9 +443,9 @@ def test_the_arm_judges_send_no_binary_answer_to_a_judge(): @pytest.mark.parametrize("command", [ - ["pairwise", "--baseline-arm", "without_docs"], - ["checklist", "--baseline-arm", "without_docs"], - ["cross-judge", "--judges", "amazon-bedrock/m", "--baseline-arm", "without_docs"], + ["score", "pairwise", "--baseline-arm", "without_docs"], + ["score", "checklist", "--baseline-arm", "without_docs"], + ["judge", "agreement", "--judges", "amazon-bedrock/m", "--baseline-arm", "without_docs"], ]) def test_the_arm_judges_refuse_a_binary_answers_file_before_any_call(tmp_path, monkeypatch, capsys, command): """Base: no judge call and exit 2 (pairwise, checklist) or an empty table; never paid calls.""" @@ -461,7 +461,7 @@ def test_the_arm_judges_refuse_a_binary_answers_file_before_any_call(tmp_path, m judge = MagicMock() monkeypatch.setattr("dokimi.eval.judge.Judge", judge) monkeypatch.setattr("dokimi.commands.judge_reliability.pairwise_judge", lambda args: (judge, "stub/judge")) - extra = ["--questions", str(questions)] if command[0] == "checklist" else [] + extra = ["--questions", str(questions)] if command[1] == "checklist" else [] args = build_parser().parse_args(["eval", *command, "--answers", str(answers), *extra, "--output", str(tmp_path / "out.json")]) with pytest.raises(SystemExit) as exc: @@ -492,7 +492,7 @@ def test_cross_judge_on_a_legacy_inline_score_run_has_nothing_to_compare(tmp_pat path = tmp_path / "run.json" path.write_text(json.dumps(_legacy_arms_run()), encoding="utf-8") - args = build_parser().parse_args(["eval", "cross-judge", "--answers", str(path), + args = build_parser().parse_args(["eval", "judge", "agreement", "--answers", str(path), "--output", str(tmp_path / "cj.json")]) with pytest.raises(SystemExit) as exc: args.func(args) @@ -516,7 +516,7 @@ def test_cross_judge_exits_2_on_a_malformed_arms_file(tmp_path, monkeypatch, cap answers = tmp_path / "arms.json" answers.write_text(json.dumps({"answers": [{"arms": {"skill": {"answer": "s"}}}]}), encoding="utf-8") - monkeypatch.setattr("sys.argv", ["dokimi", "eval", "cross-judge", "--answers", str(answers)]) + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "judge", "agreement", "--answers", str(answers)]) with pytest.raises(SystemExit) as exc: cli.main() assert exc.value.code == 2 # bad input; 1 is the verdict code of the gates @@ -544,9 +544,9 @@ def loads(text, *args, **kwargs): if content is not None: bad.write_text(content, encoding="utf-8") if flag == "--answers": - argv = ["dokimi", "eval", "cross-judge", "--answers", str(bad), "--no-stored"] + argv = ["dokimi", "eval", "judge", "agreement", "--answers", str(bad), "--no-stored"] else: - argv = ["dokimi", "eval", "cross-judge", "--answers", str(arms), "--no-stored", flag, str(bad)] + argv = ["dokimi", "eval", "judge", "agreement", "--answers", str(arms), "--no-stored", flag, str(bad)] monkeypatch.setattr("sys.argv", argv) with pytest.raises(SystemExit) as exc: cli.main() @@ -561,7 +561,7 @@ def loads(text, *args, **kwargs): assert "cannot read" in err -@pytest.mark.parametrize("command", [["cross-judge"], ["checklist"], ["judge-wording"]], +@pytest.mark.parametrize("command", [["judge", "agreement"], ["score", "checklist"], ["judge", "wording"]], ids=["cross-judge", "checklist", "judge-wording"]) @pytest.mark.parametrize("content", [None, "{not json"], ids=["missing", "malformed"]) def test_every_answers_reader_exits_2_on_an_unreadable_answers_file(tmp_path, monkeypatch, capsys, command, content): @@ -572,8 +572,8 @@ def test_every_answers_reader_exits_2_on_an_unreadable_answers_file(tmp_path, mo bad.write_text(content, encoding="utf-8") questions = tmp_path / "q.json" questions.write_text(json.dumps({"questions": []}), encoding="utf-8") - extra = ["--questions", str(questions)] if command[0] == "checklist" else [] - monkeypatch.setattr("sys.argv", ["dokimi", "eval", command[0], "--answers", str(bad), *extra]) + extra = ["--questions", str(questions)] if command[1] == "checklist" else [] + monkeypatch.setattr("sys.argv", ["dokimi", "eval", *command, "--answers", str(bad), *extra]) with pytest.raises(SystemExit) as exc: cli.main() assert exc.value.code == 2 @@ -605,7 +605,7 @@ def test_cross_judge_exits_2_on_a_validity_file_without_a_summary_object(tmp_pat encoding="utf-8") validity = tmp_path / "validity.json" validity.write_text(json.dumps(content), encoding="utf-8") - monkeypatch.setattr("sys.argv", ["dokimi", "eval", "cross-judge", "--answers", str(arms), "--no-stored", + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "judge", "agreement", "--answers", str(arms), "--no-stored", "--validity", str(validity)]) with pytest.raises(SystemExit) as exc: cli.main() @@ -627,7 +627,7 @@ def test_cross_judge_exits_2_on_a_result_file_of_the_wrong_shape(tmp_path, monke encoding="utf-8") wrong = tmp_path / "wrong.json" wrong.write_text(json.dumps(content), encoding="utf-8") - monkeypatch.setattr("sys.argv", ["dokimi", "eval", "cross-judge", "--answers", str(arms), "--no-stored", + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "judge", "agreement", "--answers", str(arms), "--no-stored", flag, str(wrong)]) with pytest.raises(SystemExit) as exc: cli.main() @@ -653,7 +653,7 @@ def test_cross_judge_names_a_result_file_measure_after_its_stem_without_a_usable result = tmp_path / "result.json" result.write_text(json.dumps({**doc, **({} if summary is None else {"summary": summary})}), encoding="utf-8") out = tmp_path / "out.json" - monkeypatch.setattr("sys.argv", ["dokimi", "eval", "cross-judge", "--answers", str(arms), "--no-stored", + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "judge", "agreement", "--answers", str(arms), "--no-stored", flag, str(result), "--output", str(out)]) cli.main() assert " · result |" in out.with_suffix(".md").read_text(encoding="utf-8") # named after result.json @@ -669,7 +669,7 @@ def test_cross_judge_names_a_result_file_measure_after_its_summary_judge(tmp_pat result = tmp_path / "result.json" result.write_text(json.dumps({**doc, "summary": {"judge": "judge-a"}}), encoding="utf-8") out = tmp_path / "out.json" - monkeypatch.setattr("sys.argv", ["dokimi", "eval", "cross-judge", "--answers", str(arms), "--no-stored", + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "judge", "agreement", "--answers", str(arms), "--no-stored", flag, str(result), "--output", str(out)]) cli.main() md = out.with_suffix(".md").read_text(encoding="utf-8") @@ -739,7 +739,7 @@ def test_cross_judge_names_a_result_entry_of_the_wrong_shape_before_any_judge_ca def test_cross_judge_takes_a_checklist_arm_with_a_null_coverage(tmp_path, monkeypatch): - """`eval checklist` writes a null coverage for an arm with no scored point.""" + """`eval score checklist` writes a null coverage for an arm with no scored point.""" from dokimi import cli arms = tmp_path / "arms.json" @@ -749,7 +749,7 @@ def test_cross_judge_takes_a_checklist_arm_with_a_null_coverage(tmp_path, monkey result.write_text(json.dumps(_checklist({"question_id": "q1", "arms": {"skill": {"coverage": None}}})), encoding="utf-8") out = tmp_path / "out.json" - monkeypatch.setattr("sys.argv", ["dokimi", "eval", "cross-judge", "--answers", str(arms), "--no-stored", + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "judge", "agreement", "--answers", str(arms), "--no-stored", "--checklist", str(result), "--output", str(out)]) cli.main() assert " · result |" in out.with_suffix(".md").read_text(encoding="utf-8") diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index 983b1a09c..7ef9ba99c 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Tests for the reference-check grader (``dokimi.eval.reference_checks``), ``eval reference`` +"""Tests for the reference-check grader (``dokimi.eval.reference_checks``), ``eval score reference`` (``dokimi.eval.reference_run``) and native-log import (``task_formats.imports.native_logs``). Native semantics are the upstream ones the importers cite: Inspect ``match_str`` / ``exact`` / @@ -269,12 +269,12 @@ def test_validate_grading_names_the_problem(): ({"checks": [{"type": "match", "values": ["x"], "weight": -1}]}, "weight must not be negative"), ]) def test_validate_grading_catches_what_grading_would_raise_on(block, problem): - # Each of these used to pass validation and then abort `eval reference` mid-run. + # Each of these used to pass validation and then abort `eval score reference` mid-run. found = validate_grading(block) assert len(found) == 1 and problem in found[0] -# --- eval reference ------------------------------------------------------------------------------ +# --- eval score reference ------------------------------------------------------------------------------ _QUESTIONS = [ {"id": "1", "question": "Capital of France?", "grading": {"checks": [ diff --git a/dokimi/tests/test_repo_builtin_graders.py b/dokimi/tests/test_repo_builtin_graders.py index e877d88c8..ab0cccfcd 100644 --- a/dokimi/tests/test_repo_builtin_graders.py +++ b/dokimi/tests/test_repo_builtin_graders.py @@ -342,7 +342,7 @@ def _repo(tmp_path, extra=None): def _check(root, out, runner, monkeypatch, *tasks): monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: runner) - argv = ["repo", "tasks", "check", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", + argv = ["eval", "check", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--out", str(out), "--cache-root", str(out / "caches")] for t in tasks: argv += ["--task", t] diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index d1de96d3d..130c3de0d 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -371,8 +371,8 @@ def test_the_check_results_artifact_keeps_the_stderr_files(): def _tasks_check_argv(tmp_path: Path, inputs: dict[str, bool]) -> list[str]: - """Run the ``repo tasks check`` step with a stub ``dokimi`` and return its argv.""" - step = next(s for s in _steps() if s.get("name") == "repo tasks check") + """Run the ``eval check`` step with a stub ``dokimi`` and return its argv.""" + step = next(s for s in _steps() if s.get("name") == "eval check") values = {f"inputs.{k}": str(v).lower() for k, v in inputs.items()} values |= {"inputs.manifest": "m.yml", "matrix.task": "t1"} env = {**os.environ, "RUNNER_TEMP": str(tmp_path)} @@ -402,6 +402,6 @@ def test_allow_unsupported_is_a_boolean_input_that_is_off_by_default(): @pytest.mark.parametrize("allow", [False, True]) def test_the_tasks_check_step_forwards_warm_and_allow_unsupported(tmp_path, warm, allow): argv = _tasks_check_argv(tmp_path, {"warm": warm, "allow-unsupported": allow}) - assert argv[:6] == ["repo", "tasks", "check", ".", "--manifest", "m.yml"] + assert argv[:5] == ["eval", "check", ".", "--manifest", "m.yml"] assert ("--warm" in argv) is warm assert ("--allow-unsupported" in argv) is allow diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index de2e3edf4..cab6f3190 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""task.json `host` requirements: probe, `repo tasks check`, `repo run`, `repo report` (contract §3, §4.3). +"""task.json `host` requirements: probe, `eval check`, `repo run`, `repo report` (contract §3, §4.3). No docker, no LLM: the probe container is answered by a fake runner, or run on this host. """ @@ -180,7 +180,7 @@ def _cli(argv, monkeypatch, runner): def _check(root, out, runner, monkeypatch, *extra): - _cli(["repo", "tasks", "check", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--out", str(out), + _cli(["eval", "check", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--out", str(out), "--cache-root", str(out / "caches"), *extra], monkeypatch, runner) @@ -258,7 +258,7 @@ def test_allow_unsupported_help_says_a_check_with_no_ok_task_still_exits_1(capsy from dokimi.cli import build_parser with pytest.raises(SystemExit): - build_parser().parse_args(["repo", "tasks", "check", "--help"]) + build_parser().parse_args(["eval", "check", "--help"]) assert "and so does a check in which no task is ok" in " ".join(capsys.readouterr().out.split()) diff --git a/dokimi/tests/test_repo_progress.py b/dokimi/tests/test_repo_progress.py index 311bd542b..a9d9b3c75 100644 --- a/dokimi/tests/test_repo_progress.py +++ b/dokimi/tests/test_repo_progress.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Progress lines of `repo tasks check` / `repo run` are flushed one by one, not at exit.""" +"""Progress lines of `eval check` / `repo run` are flushed one by one, not at exit.""" import select import subprocess diff --git a/dokimi/tests/test_repo_replay.py b/dokimi/tests/test_repo_replay.py index c21ddc9d5..92758a3f8 100644 --- a/dokimi/tests/test_repo_replay.py +++ b/dokimi/tests/test_repo_replay.py @@ -270,7 +270,7 @@ def test_the_tasks_pass_the_self_check_and_apply_under_guidance_none(hist, tmp_p _commit(root, "replay tasks") monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) out = tmp_path / "chk" - args = build_parser().parse_args(["repo", "tasks", "check", f"{root}@HEAD", "--out", str(out), + args = build_parser().parse_args(["eval", "check", f"{root}@HEAD", "--out", str(out), "--cache-root", str(out / "caches"), "--task", "review_replay_pr7"]) args.func(args) res = json.loads((out / "check/check.json").read_text())["tasks"][0] diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 827496e88..ad327757a 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -550,7 +550,7 @@ def test_worktree_image_tag_follows_the_build_context(setup, capsys): root, runner = setup["root"], setup["runner"] def check(): - argv = ["repo", "tasks", "check", str(root), "--manifest", ".agent-eval/repo-eval.yaml", + argv = ["eval", "check", str(root), "--manifest", ".agent-eval/repo-eval.yaml", "--out", str(setup["out"] / "chk"), "--cache-root", str(setup["out"] / "caches"), "--task", "fix_add"] args = build_parser().parse_args(argv) args.func(args) @@ -570,7 +570,7 @@ def test_the_image_build_gets_the_build_timeout(setup): root, runner = setup["root"], setup["runner"] def check(*extra): - argv = ["repo", "tasks", "check", str(root), "--manifest", ".agent-eval/repo-eval.yaml", + argv = ["eval", "check", str(root), "--manifest", ".agent-eval/repo-eval.yaml", "--out", str(setup["out"] / "chk"), "--cache-root", str(setup["out"] / "caches"), "--task", "fix_add", "--rebuild-image", *extra] args = build_parser().parse_args(argv) @@ -580,7 +580,7 @@ def check(*extra): check("--build-timeout", "36000") assert runner.build_timeouts == [BUILD_TIMEOUT_S, 36000] for parser_argv in (["repo", "run", str(root), "--arm", "guidance:none", "--model", "m", "--batch", "b"], - ["repo", "tasks", "check", str(root)]): + ["eval", "check", str(root)]): assert build_parser().parse_args(parser_argv).build_timeout == BUILD_TIMEOUT_S diff --git a/dokimi/tests/test_repo_runner.py b/dokimi/tests/test_repo_runner.py index 2493923e7..7efebf3da 100644 --- a/dokimi/tests/test_repo_runner.py +++ b/dokimi/tests/test_repo_runner.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Workspace preparation, containers and `repo tasks check` (contract §3, §4.3). No docker, no LLM.""" +"""Workspace preparation, containers and `eval check` (contract §3, §4.3). No docker, no LLM.""" from __future__ import annotations @@ -303,7 +303,7 @@ def _cli(argv, monkeypatch, runner): def _check_argv(root, out, *extra): - return ["repo", "tasks", "check", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", + return ["eval", "check", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--out", str(out), "--cache-root", str(out / "caches"), *extra] diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py index c099ec173..125a5b663 100644 --- a/dokimi/tests/test_repo_skill_arms.py +++ b/dokimi/tests/test_repo_skill_arms.py @@ -277,7 +277,7 @@ def test_bad_skill_arms_are_refused_before_any_cell(setup, capsys): def test_tasks_check_validates_skill_arms(setup, capsys): - argv = ["repo", "tasks", "check", f"{setup['root']}@HEAD", "--out", str(setup["out"] / "chk"), + argv = ["eval", "check", f"{setup['root']}@HEAD", "--out", str(setup["out"] / "chk"), "--cache-root", str(setup["out"] / "caches"), "--arm", "skill:skills/nope"] args = build_parser().parse_args(argv) with pytest.raises(SystemExit): @@ -483,8 +483,8 @@ def test_a_skill_the_task_base_already_ships_is_refused_before_any_cell(tmp_path task = Task("t", {"base": base}, root) with pytest.raises(ArmError, match="base of task t already has .claude/skills/lint"): check_arms([parse_arm("skill:skills/lint")], root / ".agent-eval", [task], root / ".git") - with pytest.raises(SystemExit): # `repo tasks check --arm` goes through the same check - args = build_parser().parse_args(["repo", "tasks", "check", f"{root}@HEAD", "--out", str(tmp_path / "chk"), + with pytest.raises(SystemExit): # `eval check --arm` goes through the same check + args = build_parser().parse_args(["eval", "check", f"{root}@HEAD", "--out", str(tmp_path / "chk"), "--cache-root", str(tmp_path / "caches"), "--arm", "skill:skills/lint"]) args.func(args) assert "already has .claude/skills/lint" in capsys.readouterr().err diff --git a/dokimi/tests/test_session_runner.py b/dokimi/tests/test_session_runner.py index a72d01f01..07fafbc6e 100644 --- a/dokimi/tests/test_session_runner.py +++ b/dokimi/tests/test_session_runner.py @@ -486,9 +486,9 @@ def test_a_whitespace_reply_ends_the_session_as_an_empty_one(fake_llm): @pytest.mark.parametrize("cmd", [ - ["eval", "pairwise"], - ["eval", "checklist", "--questions", "{questions}"], - ["eval", "cross-judge", "--no-stored", "--judges", "anthropic/claude-haiku-4-5"], + ["eval", "score", "pairwise"], + ["eval", "score", "checklist", "--questions", "{questions}"], + ["eval", "judge", "agreement", "--no-stored", "--judges", "anthropic/claude-haiku-4-5"], ]) def test_a_session_artifact_is_refused_by_every_judge_command(cmd, fake_llm, tmp_path, monkeypatch, capsys): diff --git a/dokimi/tests/test_skill_selection.py b/dokimi/tests/test_skill_selection.py index df3f26e66..2c62054b4 100644 --- a/dokimi/tests/test_skill_selection.py +++ b/dokimi/tests/test_skill_selection.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Skill selection: the skill-catalog: arm, its scoring, and `catalog trigger-set`.""" +"""Skill selection: the skill-catalog: arm, its scoring, and `eval triggers generate`.""" from __future__ import annotations @@ -221,19 +221,19 @@ def test_trigger_set_cli(tmp_path, catalog, capsys): {"id": "2", "text": "fix my oneDAL kmeans", "metadata": {"skill_under_test": None}}, ]}), encoding="utf-8") out = tmp_path / "set.json" - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--questions", f"{qfile}=@metadata.skill_under_test", "--out", str(out)]) data = json.loads(out.read_text(encoding="utf-8")) assert [q["expected_skill"] for q in data["queries"]] == ["linux-perf", None] assert '"exam_framing": 1' in capsys.readouterr().out with pytest.raises(SystemExit): - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--strict", + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--strict", "--questions", f"{qfile}=@metadata.skill_under_test", "--out", str(out)]) with pytest.raises(SystemExit, match="PATH=LABEL"): - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--questions", str(qfile), "--out", str(out)]) with pytest.raises(SystemExit, match="--target"): - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--format", "skill-creator", + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--format", "skill-creator", "--questions", f"{qfile}=none", "--out", str(out)]) @@ -258,7 +258,7 @@ def test_trigger_set_cli_rejects_a_label_path_that_never_resolves(tmp_path, cata {"id": "1", "text": "perf shows low IPC", "metadata": {"skill_under_test": "linux-perf"}}, ]}), encoding="utf-8") with pytest.raises(SystemExit, match="resolves for none of 1 questions") as exc: - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--questions", f"{qfile}=@metadata.skil_under_test", "--out", str(tmp_path / "o.json")]) assert exc.value.code == 2 # a path typo is an input refusal (docs/quickstart.md) @@ -286,7 +286,7 @@ def test_trigger_set_exits_2_with_a_message_on_an_unreadable_questions_file( elif content is not None: qfile.write_bytes(content) out = tmp_path / "o.json" - monkeypatch.setattr("sys.argv", ["dokimi", "catalog", "trigger-set", "--catalog", str(catalog), + monkeypatch.setattr("sys.argv", ["dokimi", "eval", "triggers", "generate", "--catalog", str(catalog), "--questions", f"{qfile}=none", "--out", str(out)]) with pytest.raises(SystemExit) as exc: cli.main() @@ -381,12 +381,12 @@ def test_trigger_set_cli_paths_with_equals_and_no_write_on_bad_label(tmp_path, c qfile = d / "q.json" qfile.write_text(json.dumps({"questions": [{"id": "1", "text": "x"}]}), encoding="utf-8") out = tmp_path / "o.json" - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--questions", f"{qfile}=linux-perf", "--out", str(out)]) assert out.exists() out.unlink() with pytest.raises(SystemExit): - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--questions", f"{qfile}=linux-pref", "--out", str(out)]) assert not out.exists() @@ -452,14 +452,14 @@ def test_trigger_set_cli_checks_each_source_and_inputs(tmp_path, catalog, capsys good = _qfile(tmp_path, "good.json", [{"id": "1", "text": "x", "metadata": {"skill_under_test": "linux-perf"}}]) typo = _qfile(tmp_path, "typo.json", [{"id": "2", "text": "y", "metadata": {"skill": "linux-perf"}}]) out = tmp_path / "o.json" - base = ["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(out)] + base = ["eval", "triggers", "generate", "--catalog", str(catalog), "--out", str(out)] with pytest.raises(SystemExit, match=r"typo\.json=@metadata\.skill_under_test: .* none of 1"): _run_cli(base + ["--questions", f"{good}=@metadata.skill_under_test", "--questions", f"{typo}=@metadata.skill_under_test"]) with pytest.raises(SystemExit, match="--rewrite needs --model"): _run_cli(base + ["--questions", f"{good}=none", "--rewrite"]) with pytest.raises(SystemExit, match="not a directory"): - _run_cli(["catalog", "trigger-set", "--catalog", str(tmp_path / "nope"), + _run_cli(["eval", "triggers", "generate", "--catalog", str(tmp_path / "nope"), "--questions", f"{good}=none", "--out", str(out)]) bad = tmp_path / "bad.json" bad.write_text(json.dumps({"questions": ["just a string"]}), encoding="utf-8") @@ -475,7 +475,7 @@ def test_trigger_set_cli_writes_each_format(tmp_path, catalog, capsys): q = _qfile(tmp_path, "q.json", [{"id": "1", "text": "profile it", "skills": ["linux-perf", "dpnp-io"]}, {"id": "2", "text": "unrelated"}]) out = tmp_path / "o.json" - base = ["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(out), + base = ["eval", "triggers", "generate", "--catalog", str(catalog), "--out", str(out), "--questions", f"{q}=@skills"] _run_cli(base + ["--format", "skill-creator", "--target", "linux-perf"]) assert json.loads(out.read_text()) == [{"query": "profile it", "should_trigger": True}] @@ -492,7 +492,7 @@ def test_trigger_set_cli_exits_with_a_message_when_out_cannot_be_written(tmp_pat out = tmp_path / "taken" out.mkdir() # a directory where the file should go with pytest.raises(SystemExit, match=rf"Error: cannot write --out {re.escape(str(out))}: ") as exc: - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(out), + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--out", str(out), "--questions", f"{q}=@skills", "--format", fmt, "--target", "linux-perf"]) assert exc.value.code == 2 assert list(out.iterdir()) == [] @@ -508,7 +508,7 @@ def reject(*_a, **_k): monkeypatch.setattr(artifact, "save_questions_artifact", reject) q = _qfile(tmp_path, "q.json", [{"id": "1", "text": "profile it", "skills": ["linux-perf"]}]) with pytest.raises(SystemExit, match=r"Error: questions set rejected: record 1: bad shape") as exc: - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(tmp_path / "o.json"), + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--out", str(tmp_path / "o.json"), "--questions", f"{q}=@skills", "--format", "questions"]) assert "cannot write" not in str(exc.value) @@ -517,7 +517,7 @@ def reject(*_a, **_k): def test_trigger_set_cli_exits_with_a_message_when_out_is_empty(tmp_path, catalog, fmt): q = _qfile(tmp_path, "q.json", [{"id": "1", "text": "profile it", "skills": ["linux-perf"]}]) with pytest.raises(SystemExit, match=r"Error: cannot write --out : ") as exc: - _run_cli(["catalog", "trigger-set", "--catalog", str(catalog), "--out", "", + _run_cli(["eval", "triggers", "generate", "--catalog", str(catalog), "--out", "", "--questions", f"{q}=@skills", "--format", fmt, "--target", "linux-perf"]) assert exc.value.code == 2 @@ -668,7 +668,7 @@ def test_trigger_set_reads_a_prompt_field(catalog): # ── trigger-set CLI: refuse input that would write a silently wrong set ── def _ts(catalog, out, *extra): - return ["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(out), *extra] + return ["eval", "triggers", "generate", "--catalog", str(catalog), "--out", str(out), *extra] def test_trigger_set_cli_rejects_an_unknown_skill_creator_target(tmp_path, catalog): diff --git a/dokimi/tests/test_task_format_corpus.py b/dokimi/tests/test_task_format_corpus.py index d2d44fb31..ac0a742de 100644 --- a/dokimi/tests/test_task_format_corpus.py +++ b/dokimi/tests/test_task_format_corpus.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""`tasks inspect` over real files of every format (tests/fixtures/task_formats/). +"""`eval inspect` over real files of every format (tests/fixtures/task_formats/). MANIFEST.yaml says what each file must be reported as; the whole corpus is one mixed tree, inspected once. Every report must validate as task_inspect.v1, and the diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py index 7ab7d3d1c..f3d2fbba4 100644 --- a/dokimi/tests/test_task_format_detect.py +++ b/dokimi/tests/test_task_format_detect.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Tests for eval-task format detection (``dokimi.task_formats``, ``tasks inspect``). +"""Tests for eval-task format detection (``dokimi.task_formats``, ``eval inspect``). Each fixture is hand-written from the upstream shape it imitates: @@ -391,16 +391,16 @@ def _run(argv: list[str]) -> None: def test_cli_inspect_prints_a_table_line_per_item(tmp_path, capsys): _harbor(tmp_path) _jsonl(tmp_path / "m.jsonl", [{"input": _CHAT, "ideal": "4"}]) - _run(["tasks", "inspect", str(tmp_path)]) + _run(["eval", "inspect", str(tmp_path)]) out = capsys.readouterr().out.splitlines() assert out[0] == f"{tmp_path / 'hb'}\tharbor\texecutable\t-> harbor, docker-*" - assert f"{tmp_path / 'm.jsonl'}\topenai-evals\tanswer-only\t1 item\t-> questions import" in out + assert f"{tmp_path / 'm.jsonl'}\topenai-evals\tanswer-only\t1 item\t-> eval import" in out assert out[-1] == "2 items: harbor 1, openai-evals 1" def test_cli_inspect_json(tmp_path, capsys): _tb1(tmp_path) - _run(["tasks", "inspect", str(tmp_path), "--json"]) + _run(["eval", "inspect", str(tmp_path), "--json"]) payload = json.loads(capsys.readouterr().out) assert [i["format"] for i in payload["items"]] == ["terminal-bench-v1"] assert payload["items"][0]["consumers"] == ["docker-*", "harbor", "benchflow"] @@ -408,7 +408,7 @@ def test_cli_inspect_json(tmp_path, capsys): def test_cli_inspect_shows_ambiguity_and_problems(tmp_path, capsys): _write(tmp_path / "x.jsonl", '{"input": "a", "target": "b", "ideal": "b"}\n{bad\n') - _run(["tasks", "inspect", str(tmp_path / "x.jsonl")]) + _run(["eval", "inspect", str(tmp_path / "x.jsonl")]) out = capsys.readouterr().out assert " ambiguous: also openai-evals" in out assert " problem: line 2: not valid JSON" in out @@ -416,7 +416,7 @@ def test_cli_inspect_shows_ambiguity_and_problems(tmp_path, capsys): def test_cli_inspect_missing_path_exits_with_a_message(tmp_path): with pytest.raises(SystemExit) as excinfo: - _run(["tasks", "inspect", str(tmp_path / "nope")]) + _run(["eval", "inspect", str(tmp_path / "nope")]) assert "no such file or directory" in str(excinfo.value) @@ -425,7 +425,7 @@ def test_cli_inspect_json_is_a_stamped_artifact_and_out_writes_the_same(tmp_path _tb1(tmp_path) out = tmp_path / "report.json" - _run(["tasks", "inspect", str(tmp_path), "--json", "--out", str(out)]) + _run(["eval", "inspect", str(tmp_path), "--json", "--out", str(out)]) printed = json.loads(capsys.readouterr().out) assert printed["schema_version"] == "task_inspect.v1" validate_artifact("task_inspect", printed) @@ -436,7 +436,7 @@ def test_cli_inspect_out_to_an_unwritable_path_exits_with_a_message(tmp_path): _tb1(tmp_path) (tmp_path / "f").write_text("") with pytest.raises(SystemExit) as excinfo: - _run(["tasks", "inspect", str(tmp_path), "--out", str(tmp_path / "f" / "r.json")]) + _run(["eval", "inspect", str(tmp_path), "--out", str(tmp_path / "f" / "r.json")]) assert "cannot write" in str(excinfo.value) @@ -600,7 +600,7 @@ def test_cli_inspect_out_writes_yaml_dates_sets_and_nan_as_plain_json(tmp_path, _write(task / "task.yaml", "instruction: do it\ndifficulty: 2024-01-01\n" "tags: !!set {b: null, a: null}\ncategory: .nan\n") out = tmp_path / "report.json" - _run(["tasks", "inspect", str(tmp_path), "--json", "--out", str(out)]) + _run(["eval", "inspect", str(tmp_path), "--json", "--out", str(out)]) printed = json.loads(capsys.readouterr().out) def refuse(name): diff --git a/dokimi/tests/test_tasks_command.py b/dokimi/tests/test_tasks_command.py index 9145951b4..d7e975181 100644 --- a/dokimi/tests/test_tasks_command.py +++ b/dokimi/tests/test_tasks_command.py @@ -73,14 +73,14 @@ def test_tasks_run_out_md_creates_missing_parent_dir(tmp_path): def test_tasks_list_reads_the_given_root(tmp_path, capsys): _fixture_task(tmp_path) parser = build_parser() - args = parser.parse_args(["tasks", "list", "--tasks-root", str(tmp_path)]) + args = parser.parse_args(["eval", "tasks", "list", "--tasks-root", str(tmp_path)]) args.func(args) assert capsys.readouterr().out.splitlines() == ["toy-task\t?\ttoy"] def test_tasks_list_without_root_exits_with_a_readable_message(): parser = build_parser() - args = parser.parse_args(["tasks", "list"]) + args = parser.parse_args(["eval", "tasks", "list"]) with pytest.raises(SystemExit) as excinfo: args.func(args) assert "--tasks-root" in str(excinfo.value) @@ -156,7 +156,7 @@ def test_tasks_run_with_an_unwritable_out_md_keeps_the_artifact_and_exits_with_a def test_tasks_list_rejects_a_root_that_is_not_a_directory(tmp_path): """A mistyped --tasks-root must not read as a suite with zero tasks.""" parser = build_parser() - args = parser.parse_args(["tasks", "list", "--tasks-root", str(tmp_path / "typo")]) + args = parser.parse_args(["eval", "tasks", "list", "--tasks-root", str(tmp_path / "typo")]) with pytest.raises(SystemExit) as excinfo: args.func(args) assert "--tasks-root is not a directory" in str(excinfo.value) @@ -165,7 +165,7 @@ def test_tasks_list_rejects_a_root_that_is_not_a_directory(tmp_path): def test_tasks_list_prints_tasks_from_a_named_root(tmp_path, capsys): _fixture_task(tmp_path) parser = build_parser() - args = parser.parse_args(["tasks", "list", "--tasks-root", str(tmp_path)]) + args = parser.parse_args(["eval", "tasks", "list", "--tasks-root", str(tmp_path)]) args.func(args) out = capsys.readouterr().out assert "toy-task" in out @@ -257,7 +257,7 @@ def invalid(self): def test_tasks_inspect_of_a_missing_path_exits_with_an_error_prefix(tmp_path): - args = build_parser().parse_args(["tasks", "inspect", str(tmp_path / "nope")]) + args = build_parser().parse_args(["eval", "inspect", str(tmp_path / "nope")]) with pytest.raises(SystemExit) as excinfo: args.func(args) assert str(excinfo.value).startswith("Error:") diff --git a/dokimi/tests/test_trigger_set_readers.py b/dokimi/tests/test_trigger_set_readers.py index 268bf9aee..206c24f4b 100644 --- a/dokimi/tests/test_trigger_set_readers.py +++ b/dokimi/tests/test_trigger_set_readers.py @@ -123,7 +123,7 @@ def test_cli_reads_each_format_back_and_relints_against_the_catalog(tmp_path, ca sc.write_text(json.dumps([{"query": "use linux-perf on it", "should_trigger": True}]), encoding="utf-8") out = tmp_path / "q.json" - _run(["catalog", "trigger-set", "--catalog", str(catalog), "--format", "questions", + _run(["eval", "triggers", "generate", "--catalog", str(catalog), "--format", "questions", "--trigger-set", str(reach), "--trigger-set", f"{sc}=linux-perf", "--out", str(out)]) rows = load_questions(out).questions assert [r.get("expected_skill") for r in rows] == [ @@ -139,7 +139,7 @@ def test_cli_read_back_keeps_source_text(tmp_path, catalog): "source_text": "Scenario: original text", "expected_skill": "linux-perf"}]}), encoding="utf-8") out = tmp_path / "q.json" - _run(["catalog", "trigger-set", "--catalog", str(catalog), "--format", "questions", + _run(["eval", "triggers", "generate", "--catalog", str(catalog), "--format", "questions", "--trigger-set", str(src), "--out", str(out)]) assert load_questions(out).questions[0]["source_text"] == "Scenario: original text" queries = ts.build_trigger_set([(ts.to_questions(ts.from_questions(json.loads( @@ -154,7 +154,7 @@ def test_cli_reach_read_back_keeps_kind_and_notes(tmp_path, catalog): "expected_skill": "linux-perf", "kind": "neighbor_negative", "notes": "near dpnp-io"}]}), encoding="utf-8") out, q_out = tmp_path / "r.json", tmp_path / "q.json" - base = ["catalog", "trigger-set", "--catalog", str(catalog), "--trigger-set", str(src)] + base = ["eval", "triggers", "generate", "--catalog", str(catalog), "--trigger-set", str(src)] _run(base + ["--out", str(out)]) row = json.loads(out.read_text(encoding="utf-8"))["queries"][0] assert (row["kind"], row["notes"]) == ("neighbor_negative", "near dpnp-io") @@ -168,7 +168,7 @@ def test_cli_refuses_bad_trigger_set_specs(tmp_path, catalog): reach = tmp_path / "r.json" reach.write_text(json.dumps({"queries": [{"id": "1", "text": "a", "expected_skill": None}]}), encoding="utf-8") - base = ["catalog", "trigger-set", "--catalog", str(catalog), "--out", str(tmp_path / "o.json")] + base = ["eval", "triggers", "generate", "--catalog", str(catalog), "--out", str(tmp_path / "o.json")] with pytest.raises(SystemExit, match="not a skill in --catalog"): _run(base + ["--trigger-set", f"{sc}=nope"]) with pytest.raises(SystemExit, match="needs the target"): @@ -195,7 +195,7 @@ def test_task_formats_name_the_reader_as_consumer(tmp_path, queries): sc.write_text(json.dumps(ts.to_skill_creator(queries, "linux-perf")), encoding="utf-8") for path in (reach, sc): report = detect(path) - assert report.consumers == ["catalog trigger-set"], report + assert report.consumers == ["eval triggers generate"], report def test_a_trigger_set_with_no_labels_is_valid_while_a_questions_typo_is_refused(catalog): diff --git a/dokimi/tests/test_typed_scorer.py b/dokimi/tests/test_typed_scorer.py index 55e65bebc..98d4dfe9a 100644 --- a/dokimi/tests/test_typed_scorer.py +++ b/dokimi/tests/test_typed_scorer.py @@ -213,7 +213,7 @@ def test_typed_arm_scores_the_pairs_and_reports_its_cost(self, tmp_path, monkeyp {"question_id": f"q{i}", "question_text": "q?", "arms": {"baseline": {"answer": (ANSWER if i % 2 else OTHER_ANSWER) + f" {i}"}}} for i in range(6)]})) - args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + args = build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--scorer", "typed:laya", "--bases", "6"]) with pytest.raises(SystemExit): # a constant scorer ties every pair: the gate FAILs args.func(args) @@ -243,7 +243,7 @@ def test_saved_report_says_which_budget_revision_and_cost(self, tmp_path, monkey for max_len in ("0", "2048"): out = tmp_path / f"v{max_len}.json" args = build_parser().parse_args([ - "eval", "judge-validity", "--answers", str(answers), "--scorer", "typed:laya", + "eval", "judge", "validate", "--answers", str(answers), "--scorer", "typed:laya", "--bases", "6", "--typed-max-len", max_len, "--output", str(out)]) with pytest.raises(SystemExit): args.func(args) @@ -258,13 +258,13 @@ def test_saved_report_says_which_budget_revision_and_cost(self, tmp_path, monkey class TestCliWiring: - """`eval judge-validity --scorer` must default to the judge and reject junk.""" + """`eval judge validate --scorer` must default to the judge and reject junk.""" @staticmethod def _parse(*argv): from dokimi.cli import build_parser - return build_parser().parse_args(["eval", "judge-validity", + return build_parser().parse_args(["eval", "judge", "validate", "--answers", "a.json", *argv]) def test_default_arm_is_the_shipped_judge(self): @@ -347,7 +347,7 @@ class TestCliKnobs: def _parse(*argv): from dokimi.cli import build_parser - return build_parser().parse_args(["eval", "judge-validity", + return build_parser().parse_args(["eval", "judge", "validate", "--answers", "a.json", *argv]) @pytest.mark.parametrize("argv", [["--typed-max-len", "-5"], @@ -417,7 +417,7 @@ def _args(tmp_path, *argv): {"question_id": f"q{i}", "question_text": "q?", "arms": {"baseline": {"answer": (ANSWER if i % 2 else OTHER_ANSWER) + f" {i}"}}} for i in range(6)]})) - return build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + return build_parser().parse_args(["eval", "judge", "validate", "--answers", str(answers), "--scorer", "typed:laya", *argv]) def test_pairwise_mode_rejects_a_typed_scorer(self, tmp_path, monkeypatch, capsys):