From 793bc74fffedab700ea1b942f0886f9fe4311ce2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 10:00:31 -0700 Subject: [PATCH 1/5] test(catalog): skill-repo shapes through catalog scan and routing Thirteen invented repo layouts, one per shape seen in public skill catalogs (mirrors, same-name skills, disable-model-invocation, template and vendored directories, zero skills, plugins nesting, spec errors, escaping links, no headings). Each runs catalog scan and novelty routing --catalog with a stub selector: no crash, and the expected status. Bugs still open on main are strict xfails naming the fix branch that will flip them. Signed-off-by: Nikolay Petrov --- ...61031_120000_napetrov_skill-repo-shapes.md | 9 + dokimi/tests/skill_repo_shapes.py | 169 +++++++++++++ dokimi/tests/test_skill_repo_shapes.py | 231 ++++++++++++++++++ 3 files changed, 409 insertions(+) create mode 100644 dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md create mode 100644 dokimi/tests/skill_repo_shapes.py create mode 100644 dokimi/tests/test_skill_repo_shapes.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md b/dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md new file mode 100644 index 000000000..9e9fabade --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md @@ -0,0 +1,9 @@ +### Added + +- **Skill-repo shape tests** — `tests/test_skill_repo_shapes.py` runs `catalog scan` + and `novelty routing --catalog` over thirteen invented repo layouts (byte-identical + and drifted mirrors, same-name skills, `disable-model-invocation`, `template/`, + vendored directories, a zero-skill repo, `plugins/*/skills` nesting, an over-long + description, a name that is not the directory name, links that leave the skill + directory, a skill with no headings). Each shape must not crash, and the bugs + still open on `main` are strict expected failures until their fixes land. diff --git a/dokimi/tests/skill_repo_shapes.py b/dokimi/tests/skill_repo_shapes.py new file mode 100644 index 000000000..db97646ea --- /dev/null +++ b/dokimi/tests/skill_repo_shapes.py @@ -0,0 +1,169 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Synthetic skill-repo layouts, one per shape seen in public skill catalogs. + +Each shape writes a small invented repo under a directory: skill names, descriptions and +bodies are made up here, so no third-party text enters the suite. ``build_shape`` returns +the catalog root for a shape; ``PROBE_QUERIES`` are requests that name no skill in particular. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +# Over the 1024-character limit of a skill description. +LONG_DESCRIPTION = "Use when polishing widgets for a demo. " + "It covers polish steps. " * 45 + +PROBE_QUERIES = [ + "polish a widget before a demo", + "check a door latch by hand", + "lint a gizmo config", +] + + +def skill_md(name: str, description: str = "Use when polishing widgets before a demo.", *, + body: str | None = None, extra: str = "") -> str: + """A SKILL.md with frontmatter ``name``/``description``, *extra* lines, and a body.""" + if body is None: + body = f"# {name}\n\n## Steps\n\nRun the {name} steps in order.\n" + return f"---\nname: {name}\ndescription: {description}\n{extra}---\n\n{body}" + + +def _put(root: Path, rel: str, text: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _door_check(root: Path) -> None: + _put(root, "skills/door-check/SKILL.md", + skill_md("door-check", "Use when checking a door latch by hand.")) + + +def _widget_polish(root: Path) -> None: + _put(root, "skills/widget-polish/SKILL.md", skill_md("widget-polish")) + + +def byte_identical_mirror(root: Path) -> None: + text = skill_md("widget-polish") + _put(root, "skills/widget-polish/SKILL.md", text) + _put(root, "plugins/kit/skills/widget-polish/SKILL.md", text) + _door_check(root) + + +def drifted_mirror(root: Path) -> None: + _widget_polish(root) + _put(root, "plugins/kit/skills/widget-polish/SKILL.md", + skill_md("widget-polish", body="# widget-polish\n\n## Steps\n\nRun the steps, then the " + "extra step.\n")) + _door_check(root) + + +def same_name_distinct_dirs(root: Path) -> None: + _widget_polish(root) + _put(root, "packs/widget-polish/SKILL.md", + skill_md("widget-polish", "Use when buffing widget casings.", + body="# widget-polish\n\n## Buff\n\nBuff the casing.\n")) + _door_check(root) + + +def disable_model_invocation(root: Path) -> None: + _put(root, "skills/gizmo-lint/SKILL.md", + skill_md("gizmo-lint", "Use when linting a gizmo config by hand.", + extra="disable-model-invocation: true\n")) + _widget_polish(root) + _door_check(root) + + +def template_dir(root: Path) -> None: + _put(root, "template/SKILL.md", + skill_md("skill-name", "Use when starting a new skill from the scaffold.")) + _widget_polish(root) + _door_check(root) + + +def vendored_dir(root: Path) -> None: + _widget_polish(root) + _door_check(root) + _put(root, "node_modules/kit/skills/door-check/SKILL.md", + skill_md("door-check", "Use when checking a door latch by hand.")) + _put(root, ".venv/lib/python3.12/site-packages/gizmo-lint/SKILL.md", + skill_md("gizmo-lint", "Use when linting a gizmo config by hand.")) + + +def zero_skill(root: Path) -> None: + _put(root, "README.md", "# Empty catalog\n\nNo skills here yet.\n") + _put(root, "docs/guide.md", "# Guide\n\nNothing to see.\n") + + +def plugins_nesting(root: Path) -> None: + _put(root, "plugins/kit/.claude-plugin/plugin.json", '{"name": "kit", "version": "0.1.0"}\n') + _put(root, "plugins/kit/skills/widget-polish/SKILL.md", skill_md("widget-polish")) + _put(root, "plugins/kit/skills/door-check/SKILL.md", + skill_md("door-check", "Use when checking a door latch by hand.")) + _put(root, "plugins/tools/skills/gizmo-lint/SKILL.md", + skill_md("gizmo-lint", "Use when linting a gizmo config by hand.")) + + +def description_too_long(root: Path) -> None: + _put(root, "skills/widget-polish/SKILL.md", skill_md("widget-polish", LONG_DESCRIPTION)) + _door_check(root) + + +def name_not_dir(root: Path) -> None: + _put(root, "skills/widget-polish-old/SKILL.md", skill_md("widget-polish")) + _door_check(root) + + +def title_case_name(root: Path) -> None: + _put(root, "skills/widget-polish/SKILL.md", skill_md("Widget Polish")) + _door_check(root) + + +def links_escape(root: Path) -> None: + body = ("# widget-polish\n\n## Steps\n\nSee [the shared guide](../../shared/guide.md) and " + "[the tables](../../references/tables.md).\n") + _put(root, "skills/widget-polish/SKILL.md", skill_md("widget-polish", body=body)) + _put(root, "shared/guide.md", "# Shared guide\n") + _door_check(root) + + +def no_usable_headings(root: Path) -> None: + _widget_polish(root) + _put(root, "skills/door-check/SKILL.md", + skill_md("door-check", "Use when checking a door latch by hand.", + body="Check the latch by hand. Then check the hinge.\n")) + + +SHAPES: dict[str, Callable[[Path], None]] = { + "byte_identical_mirror": byte_identical_mirror, + "drifted_mirror": drifted_mirror, + "same_name_distinct_dirs": same_name_distinct_dirs, + "disable_model_invocation": disable_model_invocation, + "template_dir": template_dir, + "vendored_dir": vendored_dir, + "zero_skill": zero_skill, + "plugins_nesting": plugins_nesting, + "description_too_long": description_too_long, + "name_not_dir": name_not_dir, + "title_case_name": title_case_name, + "links_escape": links_escape, + "no_usable_headings": no_usable_headings, +} + + +def build_shape(name: str, root: Path) -> Path: + """Write the shape *name* under *root* (created) and return it, the catalog root.""" + root.mkdir(parents=True, exist_ok=True) + SHAPES[name](root) + return root + + +def probe_artifact(root: Path) -> Path: + """A skill outside every catalog, so the artifact never collides with a competitor name.""" + skill_dir = root / "probe-artifact" + _put(skill_dir.parent, "probe-artifact/SKILL.md", + skill_md("probe-artifact", "Use when probing routing against a catalog.")) + return skill_dir diff --git a/dokimi/tests/test_skill_repo_shapes.py b/dokimi/tests/test_skill_repo_shapes.py new file mode 100644 index 000000000..4dab12b6f --- /dev/null +++ b/dokimi/tests/test_skill_repo_shapes.py @@ -0,0 +1,231 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Skill-repo shapes met in public catalogs: `catalog scan` and `novelty routing` over each. + +Every shape is invented (tests/skill_repo_shapes.py). The sweep asserts no command crashes; +the expectation table asserts the status each shape should get. A shape whose bug is being +fixed in another PR is an expected failure, strict, so the mark comes off when the fix lands. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.novelty.client import LLMClient +from tests.skill_repo_shapes import PROBE_QUERIES, SHAPES, build_shape, probe_artifact + +FIXED_BY_CATALOG = "fixed by fix/rescan-catalog" +FIXED_BY_ROUTING = "fixed by fix/rescan-routing" + + +@dataclass(frozen=True) +class Expect: + scan: frozenset[int] # `catalog scan --format json` exit codes + strict: frozenset[int] # the same with --strict + routing: int # `novelty routing --catalog` exit code + field: tuple[str, ...] | None # competitor names when routing exits 0; None = not checked + + +def _ok(*field: str) -> Expect: + return Expect(frozenset({0}), frozenset({0}), 0, tuple(sorted(field))) + + +_TWO = ("door-check", "widget-polish") + +EXPECT: dict[str, Expect] = { + "byte_identical_mirror": _ok(*_TWO), + "drifted_mirror": Expect(frozenset({0}), frozenset({0}), 0, None), + "same_name_distinct_dirs": Expect(frozenset({0}), frozenset({0}), 0, None), + "disable_model_invocation": _ok(*_TWO), + "template_dir": _ok(*_TWO), + "vendored_dir": _ok(*_TWO), + "zero_skill": Expect(frozenset({1, 2}), frozenset({1, 2}), 2, None), + "plugins_nesting": _ok("door-check", "gizmo-lint", "widget-polish"), + "description_too_long": Expect(frozenset({0}), frozenset({1}), 0, ("door-check",)), + "name_not_dir": Expect(frozenset({0}), frozenset({1}), 0, ("door-check",)), + "title_case_name": Expect(frozenset({0}), frozenset({1}), 0, ("door-check",)), + "links_escape": _ok(*_TWO), + "no_usable_headings": _ok(*_TWO), +} + +# Shape -> (the dimensions a bug still breaks on intel/main, the fix that will change them). +# Dimensions: scan, strict, routing, field. Only those dimensions are expected failures. +OPEN_BUGS = { + "drifted_mirror": (("routing",), FIXED_BY_CATALOG), + "same_name_distinct_dirs": (("routing",), FIXED_BY_CATALOG), + "zero_skill": (("scan", "strict"), FIXED_BY_CATALOG), + "description_too_long": (("field",), FIXED_BY_ROUTING), + "name_not_dir": (("field",), FIXED_BY_ROUTING), + "title_case_name": (("field",), FIXED_BY_ROUTING), +} + + +def _run(argv: list[str], capsys) -> tuple[int, str, str]: + """Run one dokimi command in-process; (exit code, stdout, stderr). Crashes propagate.""" + args = build_parser().parse_args(argv) + try: + args.func(args) + except SystemExit as exc: + code = 0 if exc.code is None else exc.code + else: + code = 0 + captured = capsys.readouterr() + return code, captured.out, captured.err + + +@pytest.fixture +def fake_selector(monkeypatch): + """The model arm of routing with a stub model that never answers: no network, no cost.""" + import dokimi.commands.novelty as novelty_cmd + + def client(model, provider, role, args): + return LLMClient(model=model, provider=provider, role=role, + transport=lambda prompt, system: ("", None)) + + monkeypatch.setattr(novelty_cmd, "_client", client) + + +@pytest.fixture +def shape_paths(tmp_path): + """(catalog root, probe artifact dir, queries file) for one test.""" + queries = tmp_path / "queries.json" + queries.write_text(json.dumps(PROBE_QUERIES), encoding="utf-8") + return tmp_path / "catalog", probe_artifact(tmp_path / "artifact"), queries + + +def _scan(root: Path, capsys, *, strict: bool = False) -> tuple[int, str]: + argv = ["catalog", "scan", "--root", str(root), "--format", "json"] + if strict: + argv.append("--strict") + code, out, _ = _run(argv, capsys) + return code, out + + +def _routing(root: Path, artifact: Path, queries: Path, tmp_path: Path, capsys, + *extra: str) -> tuple[int, dict | None, str]: + out_file = tmp_path / "routing.json" + argv = ["novelty", "routing", "--artifact", f"skill:{artifact}", "--catalog", str(root), + "--queries", str(queries), "--out", str(out_file), *extra] + code, _, err = _run(argv, capsys) + payload = json.loads(out_file.read_text(encoding="utf-8")) if code == 0 else None + return code, payload, err + + +def _field(payload: dict) -> tuple[str, ...]: + return tuple(sorted(payload["field"]["competitors"])) + + +@pytest.mark.parametrize("shape", sorted(SHAPES)) +def test_every_shape_runs_without_a_crash(shape, shape_paths, tmp_path, capsys, fake_selector): + # The sweep: no uncaught exception from either command, lexical and model routing alike. + root, artifact, queries = shape_paths + build_shape(shape, root) + for strict in (False, True): + code, _ = _scan(root, capsys, strict=strict) + assert code in (0, 1, 2) + code, _, _ = _routing(root, artifact, queries, tmp_path, capsys) + assert code in (0, 2) + code, _, _ = _routing(root, artifact, queries, tmp_path, capsys, "--model", "stub-selector") + assert code in (0, 2) + + +def _observe(shape: str, shape_paths, tmp_path: Path, capsys) -> dict: + """What the two commands did on one shape: exit codes and the routing field, or None.""" + root, artifact, queries = shape_paths + build_shape(shape, root) + scan_code, _ = _scan(root, capsys) + strict_code, _ = _scan(root, capsys, strict=True) + code, payload, err = _routing(root, artifact, queries, tmp_path, capsys) + return {"scan": scan_code, "strict": strict_code, "routing": code, + "field": _field(payload) if payload is not None else None, "err": err} + + +def _holds(shape: str, observed: dict) -> dict[str, bool]: + """Per dimension, whether the observation meets the expectation for *shape*.""" + expect = EXPECT[shape] + return { + "scan": observed["scan"] in expect.scan, + "strict": observed["strict"] in expect.strict, + "routing": observed["routing"] == expect.routing, + "field": expect.field is None or observed["field"] == expect.field, + } + + +@pytest.mark.parametrize("shape", sorted(SHAPES)) +def test_every_shape_gets_its_expected_status(shape, shape_paths, tmp_path, capsys, fake_selector): + # Every dimension is strict except the one a bug in another PR still breaks (OPEN_BUGS). + observed = _observe(shape, shape_paths, tmp_path, capsys) + holds = _holds(shape, observed) + open_dims = OPEN_BUGS[shape][0] if shape in OPEN_BUGS else () + failed = [dim for dim, ok in holds.items() if not ok and dim not in open_dims] + assert not failed, f"{shape}: {failed} off expectation; observed {observed}" + + +@pytest.mark.parametrize("shape", [ + pytest.param(shape, id=shape, marks=pytest.mark.xfail(strict=True, reason=reason)) + for shape, (_, reason) in sorted(OPEN_BUGS.items()) +]) +def test_open_bug_dimension_meets_its_expectation(shape, shape_paths, tmp_path, capsys, + fake_selector): + # Strict xfail on the open dimensions only: when that fix lands, this XPASSes and the mark comes off. + observed = _observe(shape, shape_paths, tmp_path, capsys) + holds = _holds(shape, observed) + for dim in OPEN_BUGS[shape][0]: + assert holds[dim], f"{shape}.{dim}: observed {observed}" + + +def test_a_mirror_is_listed_by_scan_but_counted_once_in_the_field(shape_paths, tmp_path, capsys, + fake_selector): + root, artifact, queries = shape_paths + build_shape("byte_identical_mirror", root) + _, out = _scan(root, capsys) + rows = json.loads(out)["skills"] + assert len(rows) == 3 + assert sum(r["duplicate_of"] is not None for r in rows) == 1 + code, payload, _ = _routing(root, artifact, queries, tmp_path, capsys) + assert code == 0 and payload["n_candidates"] == 3 # the artifact and two competitors + + +def test_the_disabled_skill_is_left_out_of_the_routing_field_and_said_so(shape_paths, tmp_path, + capsys, fake_selector): + root, artifact, queries = shape_paths + build_shape("disable_model_invocation", root) + code, payload, err = _routing(root, artifact, queries, tmp_path, capsys) + assert code == 0 and "gizmo-lint" not in _field(payload) + assert "gizmo-lint" in err and "disable-model-invocation" in err + + +def test_all_vs_all_names_the_flag_when_queries_name_a_disabled_skill(shape_paths, tmp_path, + capsys): + root, _, _ = shape_paths + build_shape("disable_model_invocation", root) + by_skill = tmp_path / "by_skill.json" + by_skill.write_text(json.dumps({"gizmo-lint": ["lint a gizmo config"], + "door-check": ["check a door latch"]}), encoding="utf-8") + argv = ["novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(by_skill), "--out", str(tmp_path / "avsa.json")] + code, _, err = _run(argv, capsys) + assert code == 2 and "--include-disabled" in err + code, _, _ = _run([*argv, "--include-disabled"], capsys) + assert code == 0 + + +def test_headings_as_queries_on_a_skill_with_no_headings_is_a_usage_refusal(shape_paths, tmp_path, + capsys): + root, _, _ = shape_paths + build_shape("no_usable_headings", root) + artifact = root / "skills" / "door-check" # its only body line is prose, no heading + out_file = tmp_path / "routing.json" + argv = ["novelty", "routing", "--artifact", f"skill:{artifact}", "--catalog", str(root), + "--queries-from-headings", "--out", str(out_file)] + code, _, err = _run(argv, capsys) + assert code == 2 and "no usable queries" in err and "headings" in err + # The refusal still leaves a record, so a batch run shows why it has no rate. + record = json.loads(out_file.read_text(encoding="utf-8")) + assert record["status"] == "no_queries" and record["queries_self_derived"] is True From 30906cf6a5c2d9d766a804f66c7a9251aa10c68e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 14:20:07 +0000 Subject: [PATCH 2/5] test(catalog): pin which mirror copy scan keeps as the original Signed-off-by: Anton Volkov --- dokimi/tests/test_skill_repo_shapes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dokimi/tests/test_skill_repo_shapes.py b/dokimi/tests/test_skill_repo_shapes.py index 4dab12b6f..f4363433f 100644 --- a/dokimi/tests/test_skill_repo_shapes.py +++ b/dokimi/tests/test_skill_repo_shapes.py @@ -188,6 +188,9 @@ def test_a_mirror_is_listed_by_scan_but_counted_once_in_the_field(shape_paths, t rows = json.loads(out)["skills"] assert len(rows) == 3 assert sum(r["duplicate_of"] is not None for r in rows) == 1 + copy = next(r for r in rows if r["duplicate_of"]) + assert copy["path"].endswith("plugins/kit/skills/widget-polish") + assert copy["duplicate_of"] == "skills/widget-polish" code, payload, _ = _routing(root, artifact, queries, tmp_path, capsys) assert code == 0 and payload["n_candidates"] == 3 # the artifact and two competitors From 07c1b3c25670460645ac1145732a737270010782 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 14:20:16 +0000 Subject: [PATCH 3/5] test(catalog): limit the strict xfails to AssertionError Signed-off-by: Anton Volkov --- dokimi/tests/test_skill_repo_shapes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_skill_repo_shapes.py b/dokimi/tests/test_skill_repo_shapes.py index f4363433f..323e67023 100644 --- a/dokimi/tests/test_skill_repo_shapes.py +++ b/dokimi/tests/test_skill_repo_shapes.py @@ -168,7 +168,8 @@ def test_every_shape_gets_its_expected_status(shape, shape_paths, tmp_path, caps @pytest.mark.parametrize("shape", [ - pytest.param(shape, id=shape, marks=pytest.mark.xfail(strict=True, reason=reason)) + pytest.param(shape, id=shape, + marks=pytest.mark.xfail(strict=True, raises=AssertionError, reason=reason)) for shape, (_, reason) in sorted(OPEN_BUGS.items()) ]) def test_open_bug_dimension_meets_its_expectation(shape, shape_paths, tmp_path, capsys, From e667713f819613541a52969eac95d7f6b1a992dc Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 14:20:17 +0000 Subject: [PATCH 4/5] docs: drop the changelog fragment of a test-only change Signed-off-by: Anton Volkov --- .../20261031_120000_napetrov_skill-repo-shapes.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md diff --git a/dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md b/dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md deleted file mode 100644 index 9e9fabade..000000000 --- a/dokimi/changelog.d/20261031_120000_napetrov_skill-repo-shapes.md +++ /dev/null @@ -1,9 +0,0 @@ -### Added - -- **Skill-repo shape tests** — `tests/test_skill_repo_shapes.py` runs `catalog scan` - and `novelty routing --catalog` over thirteen invented repo layouts (byte-identical - and drifted mirrors, same-name skills, `disable-model-invocation`, `template/`, - vendored directories, a zero-skill repo, `plugins/*/skills` nesting, an over-long - description, a name that is not the directory name, links that leave the skill - directory, a skill with no headings). Each shape must not crash, and the bugs - still open on `main` are strict expected failures until their fixes land. From f71a36ceb3c9623086e8bb3d1f3261f9c1617acb Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 14:42:44 +0000 Subject: [PATCH 5/5] test(catalog): compare the mirror path as a Path, not a suffix Signed-off-by: Anton Volkov --- dokimi/tests/test_skill_repo_shapes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/tests/test_skill_repo_shapes.py b/dokimi/tests/test_skill_repo_shapes.py index 323e67023..0f01d3e0b 100644 --- a/dokimi/tests/test_skill_repo_shapes.py +++ b/dokimi/tests/test_skill_repo_shapes.py @@ -190,7 +190,7 @@ def test_a_mirror_is_listed_by_scan_but_counted_once_in_the_field(shape_paths, t assert len(rows) == 3 assert sum(r["duplicate_of"] is not None for r in rows) == 1 copy = next(r for r in rows if r["duplicate_of"]) - assert copy["path"].endswith("plugins/kit/skills/widget-polish") + assert Path(copy["path"]) == root / "plugins" / "kit" / "skills" / "widget-polish" assert copy["duplicate_of"] == "skills/widget-polish" code, payload, _ = _routing(root, artifact, queries, tmp_path, capsys) assert code == 0 and payload["n_candidates"] == 3 # the artifact and two competitors