diff --git a/dokimi/changelog.d/20261030_170000_napetrov_catalog-bundle-drift-checks.md b/dokimi/changelog.d/20261030_170000_napetrov_catalog-bundle-drift-checks.md new file mode 100644 index 00000000..35539f09 --- /dev/null +++ b/dokimi/changelog.d/20261030_170000_napetrov_catalog-bundle-drift-checks.md @@ -0,0 +1,14 @@ +### Added + +- **`catalog scan` checks each skill's bundle and description, and detects drifted mirrors** — + a relative Markdown link in `SKILL.md` to a file that does not exist is a `warn` + (`link-missing`), and says so when the file exists from the catalog root instead (a wrong + base). A link or `../` code span that reaches a file outside the skill directory is a `warn` + (`link-escapes-skill`, `path-escapes-skill`): it breaks when the skill is installed alone. + A description with no "when" clause is a `warn` (`description-no-when`), as is one + whose words overlap another skill's by at least half (`description-near-duplicate`). Two + skills in different paths with the same directory name and different `SKILL.md` text are + a `warn` on both (`mirror-drift`), naming the other copies. Links inside code fences, URLs, + anchors, and code spans that name no file are not checked. Skills with + `disable-model-invocation: true` are exempt from the description checks. Warnings never + change the exit code; a spec `error` fails only with `--strict`, while a load error always fails. diff --git a/dokimi/docs/external-content.md b/dokimi/docs/external-content.md index 4eb88f84..a6102976 100644 --- a/dokimi/docs/external-content.md +++ b/dokimi/docs/external-content.md @@ -129,6 +129,19 @@ offers; each other copy is the `warn` and names it. Warnings never change the exit code. `disable-model-invocation: true` is printed as a `note`, because the model is never offered that skill. +The scan also checks the bundle and the description. A relative Markdown link +to a file the skill does not have is a `warn` (`link-missing`), and says when the +file exists from the catalog root instead. Example links such as `[Title](URL)` +are common, so `--strict` does not fail on them. A link, or a `../` code span, +that reaches a file outside the skill directory is a `warn` +(`link-escapes-skill`, `path-escapes-skill`), since it breaks when the skill is +installed alone. A description with no "when" clause (`use when ...`, `when deploying ...`) is a `warn` +(`description-no-when`), and so is one that overlaps another skill's by at least +half (`description-near-duplicate`). Two skills in different paths with the same +directory name and different `SKILL.md` text are a `warn` on both +(`mirror-drift`), naming the other copies. Skills with +`disable-model-invocation: true` are exempt from the description checks. + Frontmatter that is not valid YAML (a bare colon in `description:`, say) stops the skill from loading, since the loader reads it as having no description. The scan reports that as a `frontmatter-parse` error carrying the YAML message, and diff --git a/dokimi/dokimi/external/validate.py b/dokimi/dokimi/external/validate.py index ff13c2c7..ab0242d2 100644 --- a/dokimi/dokimi/external/validate.py +++ b/dokimi/dokimi/external/validate.py @@ -30,11 +30,31 @@ ``disable-model-invocation: true`` is not a finding: the model is never offered that skill, so it is reported as a flag (the same reading as the skill arm's check in :mod:`dokimi.harnesses.skill_treatment`). + +Bundle links: a relative Markdown link in ``SKILL.md`` whose file does not exist is a +``warn`` (``link-missing``), inline or reference-style (``[label]: target``), with an +optional ``<...>`` destination, and URI destinations (``tel:``, ``urn:``, ``mailto:``) are +not files. A note is added when the file exists from the catalog root +instead (a wrong base). Example links such as ``[Title](URL)`` are common, so this is not +an error and does not fail ``--strict``. A link to a file outside the skill directory is a ``warn`` +(``link-escapes-skill``): it breaks when the skill is installed alone. A ``../`` code span +naming such a file is ``path-escapes-skill``. Code spans that name no file are not checked, +and neither are code fences: those are usually examples or paths in the user's own project. + +Descriptions: one with no "when" clause is a ``warn`` (``description-no-when``), +and so is one whose tokens overlap another skill's at a Jaccard similarity of at least +:data:`NEAR_DUPLICATE_JACCARD` (``description-near-duplicate``). Skills with +``disable-model-invocation: true`` are exempt from both, as the model never chooses them. + +Mirror drift: two skills in different paths with the same directory name but different +``SKILL.md`` text are a ``warn`` on both (``mirror-drift``), naming the other copies: the +same skill has probably been edited in one place and not the other. """ from __future__ import annotations import hashlib +import os import re from dataclasses import dataclass, field from pathlib import Path, PurePosixPath @@ -50,6 +70,44 @@ NAME_PATTERN = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") NAME_MAX = 64 DESCRIPTION_MAX = 1024 +NEAR_DUPLICATE_JACCARD = 0.5 + +# A "when" clause of any subject and verb ("run migrations when deploying"), or a use phrase: the model +# chooses a skill from its description alone. +WHEN_RE = re.compile( + r"\b(?:when\s+\w|use for\b|use this skill\b|trigger(?:s|ed)?\s*(?:on\b|:)|invoke only\b|activate\b|" + r"invoke this\b|use (?:it |this |these )?(?:when|if|before|after|whenever|proactively|anytime)\b|" + r"whenever\b)", + re.I, +) +# Words that say nothing about what a skill is for; left out of the description overlap. +STOP_WORDS = frozenset( + """a an the and or of to in on for with by from as at is are be been being this that + these those it its it's you your we our use used using when with without into than then + also any all each skill skills claude user users make makes do does not no yes can will + should must may might if else only other such via per about over under more most very + just""".split() +) +# An inline link target: balanced parentheses (``guide(v2).md``) or ```` with spaces; +# an optional title follows the target. +_TARGET_TITLE = r"""(?:\s+(?:"[^"]*"|'[^']*'|\([^()]*\)))?""" +LINK_RE = re.compile(rf"\]\(\s*(<[^<>\n]*>|(?:[^()\s]|\([^()\s]*\))+){_TARGET_TITLE}\s*\)") +# A reference-style link definition, ``[label]: destination``, the destination on this line or the +# next (footnotes ``[^n]:`` excluded). Matched over the whole body, not line by line. +REF_DEF_RE = re.compile( + r"""^ {0,3}\[(?!\^)[^\]\n]+\]:[ \t]*(?:\n[ \t]*)?(<[^>\n]*>|\S+)""" + r"""(?:[ \t]+(?:"[^"]*"|'[^']*'|\([^)]*\)))?[ \t]*$""", + re.MULTILINE, +) +FENCE_OPEN_RE = re.compile(r"\ {0,3}(`{3,}|~{3,})(.*)$") +FENCE_CLOSE_RE = re.compile(r"\ {0,3}(`{3,}|~{3,})[ \t]*$") +# RFC 3986 scheme, dots included; a dotted one with a line or symbol after it is a file (_file_ref). +URI_SCHEME_RE = re.compile(r"[A-Za-z][A-Za-z0-9+\-.]*:") +# After a file, a line or a symbol (``run.py:42``, ``run.py:10-20``, ``run.py:main``): read as the file's name. +FILE_SUFFIX_RE = re.compile(r"\d+(?:[-:]\d+)?|[A-Za-z_]\w*") +LOCATION_SUFFIX_RE = re.compile(r":(?:\d+(?:[-:]\d+)?|[A-Za-z_]\w*)$") +CODE_SPAN_RE = re.compile(r"`([^`\n]+)`") +SKILL_DIR_VAR = "${CLAUDE_SKILL_DIR}/" @dataclass @@ -100,7 +158,7 @@ def _frontmatter_findings(skill: Skill, *, at_root: bool = False) -> tuple[list[ meta, parse_error = _strict_meta(skill.path.read_text(encoding="utf-8")) if parse_error is not None: return [parse_error], False - dir_name = skill.path.parent.name + dir_name = _dir_name_of(skill.path.parent) findings: list[Finding] = [] declared = str(meta.get("name") or "").strip() if not declared: @@ -184,15 +242,261 @@ def _relative_to_root(path: Path, root: Path) -> str: return path.as_posix() +def _counterpart(other: tuple[SkillCatalogEntry, Skill, SkillReview], entry: SkillCatalogEntry) -> str: + """The other skill's directory relative to its root, qualified by that root when it is another checkout.""" + o_entry, o_skill, _ = other + where = _relative_to_root(o_skill.path.parent, o_entry.source_root) + if o_entry.source_root.resolve() == entry.source_root.resolve(): + return where + return f"{where} in {o_entry.source_root.as_posix()}" + + +def _outside_fences(body: str) -> list[str]: + """The lines of ``body`` that are not inside a fenced code block. + + A block closes only at a line of the same marker (backticks or tildes) that is at least as long + as the opening run, as in CommonMark. + """ + lines: list[str] = [] + fence = "" # the run that opened the current block; "" outside one + for line in body.splitlines(): + if fence: + close = FENCE_CLOSE_RE.match(line) + if close and close.group(1)[0] == fence[0] and len(close.group(1)) >= len(fence): + fence = "" + lines.append("") # a blank line keeps a definition above the block from reaching below it + continue + opened = FENCE_OPEN_RE.match(line) + # a backtick info string may not hold a backtick: "```x```" is an inline code span, not a fence + if opened and not (opened.group(1)[0] == "`" and "`" in opened.group(2)): + fence = opened.group(1) + lines.append("") + else: + lines.append(line) + return lines + + +def _file_ref(raw: str, link: bool = False) -> Optional[str]: + """The skill-relative file a reference names, or None if it names no file to check. + + None for a URL or any other URI (``tel:``, ``urn:``, ``MAILTO:``), a bare anchor, an + absolute path, or a template with a variable other than ``${CLAUDE_SKILL_DIR}``. A + ``<...>`` destination (CommonMark) is read as the path inside the brackets. A link + destination (``link``) is a URL: its ``?query`` is not part of the path. + """ + text = raw.strip() + if text.startswith("<") and text.endswith(">"): + text = text[1:-1].strip() + text = text.split("#", 1)[0].strip() + if link: + text = text.split("?", 1)[0].strip() + if not text or "://" in text or text.startswith("/"): + return None + scheme = URI_SCHEME_RE.match(text) + if scheme and not ("." in scheme.group() and FILE_SUFFIX_RE.fullmatch(text[scheme.end() :])): + return None + text = LOCATION_SUFFIX_RE.sub("", text) # the file is named before its line or symbol suffix + if not text: + return None + if text.startswith(SKILL_DIR_VAR): + text = text[len(SKILL_DIR_VAR) :] + if not text: + return None + if any(c in text for c in "${}<>*"): + return None + return text + + +def _resolved(path: Path) -> Optional[Path]: + """``path`` resolved, or None when the OS refuses the name (a NUL byte, a symlink loop).""" + try: + return path.resolve() + except (OSError, RuntimeError, ValueError): + return None + + +def _exists(path: Path) -> bool: + """``path.exists()``, where a name the OS refuses (over 255 bytes in one part, say) is absent.""" + try: + return path.exists() + except (OSError, ValueError): + return False + + +def _bundle_findings(skill: Skill, source_root: Path) -> list[Finding]: + """Link and path checks on the body of ``skill``: missing targets, or targets outside it.""" + skill_dir = skill.path.parent + real_dir = skill_dir.resolve() + lines = _outside_fences(skill.body) + links = dict.fromkeys( + [m for line in lines for m in LINK_RE.findall(line)] + + REF_DEF_RE.findall("\n".join(lines)) + ) + spans = dict.fromkeys( + s.strip() + for line in lines + for s in CODE_SPAN_RE.findall(line) + if s.strip().startswith("../") + ) + findings: list[Finding] = [] + for raw in links: + text = _file_ref(raw, link=True) + if text is None: + continue + target = _resolved(skill_dir / text) + if target is not None and _exists(target): + if not target.is_relative_to(real_dir): + findings.append( + Finding( + "warn", + "link-escapes-skill", + f"link {raw!r} points outside the skill directory; it breaks when " + "the skill is installed alone", + ) + ) + continue + from_root = "" + if _exists(source_root / text): + from_root = " (it exists from the catalog root: the base is wrong)" + findings.append( + Finding( + "warn", + "link-missing", + f"link {raw!r} names no file relative to the skill directory{from_root}", + ) + ) + for raw in spans: + text = _file_ref(raw) + if text is None: + continue + target = _resolved(skill_dir / text) + if target is not None and _exists(target) and not target.is_relative_to(real_dir): + findings.append( + Finding( + "warn", + "path-escapes-skill", + f"{raw!r} points outside the skill directory; it breaks when the skill " + "is installed alone", + ) + ) + return findings + + +def _description_findings(skill: Skill) -> list[Finding]: + """A ``warn`` when the description has no "when" clause.""" + if not WHEN_RE.search(" ".join(skill.description.split())): + return [ + Finding( + "warn", + "description-no-when", + "description has no 'when' clause (e.g. 'use when ...'); the model chooses " + "the skill from the description alone", + ) + ] + return [] + + +def _tokens(description: str) -> frozenset[str]: + """The lowercase words of three or more letters, in any script, less the stop words.""" + words = re.findall(r"[^\W\d_]{3,}", description.lower()) + return frozenset(w for w in words if w not in STOP_WORDS) + + +def _jaccard(a: frozenset[str], b: frozenset[str]) -> float: + return len(a & b) / len(a | b) if a and b else 0.0 + + +def _dir_name_of(directory: Path) -> str: + """The name of ``directory`` with its ``..`` parts taken away lexically: ``alpha/sub/..`` is ``alpha``. + + Lexical, not resolved, so a symlinked skill directory keeps the name it is listed under. + """ + return Path(os.path.normpath(directory)).name + + +def _dir_name(item: tuple[SkillCatalogEntry, Skill, SkillReview]) -> Optional[str]: + """The directory name that can make a skill a mirror of another, None for the root skill. + + The root ``SKILL.md`` is named by the checkout's directory, not by the skill, so it has no + name to match (as ``_frontmatter_findings`` treats it with ``at_root``). + """ + entry, skill, _ = item + if skill.path.parent.resolve() == entry.source_root.resolve(): + return None + return _dir_name_of(skill.path.parent) + + +def _mirror_drift(loaded: list[tuple[SkillCatalogEntry, Skill, SkillReview]]) -> None: + """``mirror-drift`` on each skill that shares its directory name with a differing copy.""" + by_dir_name: dict[str, list[tuple[SkillCatalogEntry, Skill, SkillReview]]] = {} + for item in loaded: + name = _dir_name(item) + if name is not None: + by_dir_name.setdefault(name, []).append(item) + for group in by_dir_name.values(): + for entry, _, review in group: + others = [ + _counterpart(item, entry) + for item in group + if item[2].content_sha256 != review.content_sha256 + ] + if others: + review.findings.append( + Finding( + "warn", + "mirror-drift", + f"SKILL.md differs from {', '.join(others)} (the same skill name in " + "another path); the copies may have drifted apart", + ) + ) + + +def _near_duplicates(loaded: list[tuple[SkillCatalogEntry, Skill, SkillReview]]) -> None: + """``description-near-duplicate`` on both skills of each pair whose descriptions overlap.""" + shown = [item for item in loaded if not item[2].disable_model_invocation] + tokens = [_tokens(" ".join(item[1].description.split())) for item in shown] + names = [_dir_name(item) for item in shown] + by_token: dict[str, list[int]] = {} + for index, words in enumerate(tokens): + for word in words: + by_token.setdefault(word, []).append(index) + for i, (entry, _, review) in enumerate(shown): + # only a pair that shares a word can overlap + for j in sorted({k for word in tokens[i] for k in by_token[word] if k > i}): + o_entry, _, o_review = shown[j] + if o_review.content_sha256 == review.content_sha256: + continue # an identical SKILL.md: already a duplicate + if names[i] is not None and names[i] == names[j]: + continue # the same skill in another path: already a mirror-drift + similarity = _jaccard(tokens[i], tokens[j]) + if similarity >= NEAR_DUPLICATE_JACCARD: + pairs = ( + (review, _counterpart(shown[j], entry)), + (o_review, _counterpart(shown[i], o_entry)), + ) + for side, other in pairs: + side.findings.append( + Finding( + "warn", + "description-near-duplicate", + f"description overlaps {other} (word overlap {similarity:.0%}); " + "the model may pick either", + ) + ) + + def review_skills(entries: Sequence[SkillCatalogEntry]) -> list[Optional[SkillReview]]: """One :class:`SkillReview` per entry, in order. None for an entry that did not load, unless its frontmatter is broken YAML: then its review carries the ``frontmatter-parse`` error. Of each identical ``SKILL.md``, the copy nearest the root is the original; every other copy names it in ``duplicate_of``. + Mirror drift and near-duplicate descriptions are checked across all the loaded skills, + and a skill with ``disable-model-invocation`` set is left out of the description checks. """ reviews: list[Optional[SkillReview]] = [] copies: list[tuple[SkillReview, str, str]] = [] # (review, skill directory, digest) per loaded skill + loaded: list[tuple[SkillCatalogEntry, Skill, SkillReview]] = [] for entry in entries: if not entry.loaded or entry.skill is None: reviews.append(_unloaded_review(entry)) @@ -202,10 +506,14 @@ def review_skills(entries: Sequence[SkillCatalogEntry]) -> list[Optional[SkillRe skill, at_root=skill.path.parent.resolve() == entry.source_root.resolve() ) review = SkillReview(findings=findings, disable_model_invocation=hidden) + review.findings += _bundle_findings(skill, entry.source_root) + if not hidden: + review.findings += _description_findings(skill) review.content_sha256 = _skill_digest(skill) where = _relative_to_root(skill.path.parent, entry.source_root) copies.append((review, where, review.content_sha256)) reviews.append(review) + loaded.append((entry, skill, review)) original_of: dict[str, tuple[SkillReview, str]] = {} for copy, where, digest in sorted(copies, key=lambda c: (len(PurePosixPath(c[1]).parts), c[1])): @@ -222,4 +530,6 @@ def review_skills(entries: Sequence[SkillCatalogEntry]) -> list[Optional[SkillRe f"SKILL.md is identical to {where}; the same skill is counted twice", ) ) + _mirror_drift(loaded) + _near_duplicates(loaded) return reviews diff --git a/dokimi/tests/test_catalog_links.py b/dokimi/tests/test_catalog_links.py new file mode 100644 index 00000000..9472f182 --- /dev/null +++ b/dokimi/tests/test_catalog_links.py @@ -0,0 +1,174 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Link targets, reference definitions, fenced blocks and directory names in ``dokimi.external.validate``. + +Split from ``test_catalog_validate.py``, which is over the file-length cap. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from dokimi.external import read_skill_manifest, review_skills + + +def _skill(root: Path, rel: str, name: str | None = None, description: str = "Use for things.", + extra: str = "", body: str = "# Body\n") -> Path: + """Write ``//SKILL.md``; ``name`` None leaves the key out of the frontmatter.""" + skill_dir = root / rel + skill_dir.mkdir(parents=True, exist_ok=True) + head = f"name: {name}\n" if name is not None else "" + (skill_dir / "SKILL.md").write_text( + f"---\n{head}description: {description}\n{extra}---\n{body}", encoding="utf-8") + return skill_dir + + +def _reviews_by_name(root: Path) -> dict: + entries = read_skill_manifest(root) + return {e.name: r for e, r in zip(entries, review_skills(entries), strict=True)} + + +def _codes(review) -> list[str]: + return [f.code for f in review.findings] + + +def test_a_link_destination_with_parentheses_to_an_existing_file_is_not_missing(tmp_path): + skill_dir = _skill(tmp_path / "skills", "parens", name="parens", description="Use for parens.", + body="See [the guide](references/guide(v2).md) and [a](references/a(b)(c).md).\n") + (skill_dir / "references").mkdir() + (skill_dir / "references" / "guide(v2).md").write_text("# Guide\n", encoding="utf-8") + (skill_dir / "references" / "a(b)(c).md").write_text("# A\n", encoding="utf-8") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_a_link_destination_with_parentheses_to_a_missing_file_names_the_whole_path(tmp_path): + _skill(tmp_path / "skills", "parens-missing", name="parens-missing", description="Use for parens.", + body="See [the guide](references/guide(v2).md).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["link-missing"] + assert "'references/guide(v2).md'" in review.findings[0].message + + +@pytest.mark.parametrize("exists", [True, False]) +def test_angle_bracket_destination_with_spaces_is_checked(tmp_path, exists): + (tmp_path / "shared").mkdir() + if exists: + (tmp_path / "shared" / "my notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "spaced-link", name="spaced-link", description="Use for spaced link.", + body="See [notes](<../../shared/my notes.md>).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == (["link-escapes-skill"] if exists else ["link-missing"]) + + +@pytest.mark.parametrize("exists", [True, False]) +def test_reference_definition_with_destination_on_the_next_line_is_checked(tmp_path, exists): + (tmp_path / "shared").mkdir() + if exists: + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "next-line", name="next-line", description="Use for next line.", + body="See [notes][n].\n\n[n]:\n ../../shared/notes.md\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == (["link-escapes-skill"] if exists else ["link-missing"]) + + +def test_a_tilde_line_inside_a_backtick_fence_does_not_close_it(tmp_path): + _skill(tmp_path / "skills", "fences", name="fences", description="Use for fences.", + body="```markdown\n~~~\n[example](missing.md)\n```\n[real](also-missing.md)\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["link-missing"] + assert "'also-missing.md'" in review.findings[0].message + + +def test_a_closing_fence_shorter_than_the_opening_one_does_not_close_it(tmp_path): + _skill(tmp_path / "skills", "fence-run", name="fence-run", description="Use for fence run.", + body="````\n```\n[example](missing.md)\n````\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_a_fence_indented_four_spaces_does_not_open_a_block(tmp_path): + _skill(tmp_path / "skills", "indented", name="indented", description="Use for indented.", + body=" ```\n[a](gone.md)\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["link-missing"] + + +def test_skills_named_through_a_subdirectory_dot_dot_are_not_mirror_drift(tmp_path): + # "skills/alpha/sub/.." is the skill directory "alpha": two such skills are not copies of each other. + _skill(tmp_path / "skills", "alpha", name="alpha", description="Use for alpha work.", body="# Alpha\n") + _skill(tmp_path / "skills", "beta", name="beta", description="Use for beta work.", body="# Beta\n") + (tmp_path / "skills" / "alpha" / "sub").mkdir() + (tmp_path / "skills" / "beta" / "sub").mkdir() + (tmp_path / "skills.yaml").write_text( + "skills:\n" + " - name: alpha\n catalog_path: skills/alpha/sub/..\n" + " - name: beta\n catalog_path: skills/beta/sub/..\n", + encoding="utf-8") + + reviews = _reviews_by_name(tmp_path) + assert _codes(reviews["alpha"]) == [] + assert _codes(reviews["beta"]) == [] + + +@pytest.mark.parametrize("body", [ + '[a](gone.md "t")\n', + "[a](gone.md 't')\n", + "[a](gone.md (t))\n", + "[a]( gone.md )\n", + "```x``` then\n[a](gone.md)\n", +]) +def test_link_forms_and_fence_lookalikes_are_still_checked(tmp_path, body): + _skill(tmp_path / "skills", "forms", name="forms", description="Use for forms.", body=body) + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["link-missing"] + + +def test_a_link_inside_parentheses_does_not_take_the_closing_one(tmp_path): + skill_dir = _skill(tmp_path / "skills", "wrapped", name="wrapped", description="Use for wrapped.", + body="(see [a](here.md))\n") + (skill_dir / "here.md").write_text("# Here\n", encoding="utf-8") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_a_reference_label_before_a_fence_does_not_take_a_destination_after_it(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "label-fence", name="label-fence", description="Use for label fence.", + body="See [notes][n].\n\n[n]:\n```\ncode\n```\n../../shared/notes.md\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_a_link_with_a_query_to_an_existing_file_is_checked_by_its_path(tmp_path): + skill_dir = _skill(tmp_path / "skills", "query-link", name="query-link", description="Use for query links.", + body="See [the guide](references/guide.md?raw=1).\n") + (skill_dir / "references").mkdir() + (skill_dir / "references" / "guide.md").write_text("# Guide\n", encoding="utf-8") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_a_query_bearing_link_outside_the_skill_still_escapes(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "guide.md").write_text("# Guide\n", encoding="utf-8") + _skill(tmp_path / "skills", "query-escape", name="query-escape", description="Use for query escapes.", + body="See [the guide](../../shared/guide.md?raw=1#top).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["link-escapes-skill"] diff --git a/dokimi/tests/test_catalog_validate.py b/dokimi/tests/test_catalog_validate.py index 2313223a..cce832f1 100644 --- a/dokimi/tests/test_catalog_validate.py +++ b/dokimi/tests/test_catalog_validate.py @@ -81,8 +81,9 @@ def test_name_missing_from_frontmatter_is_a_warning(tmp_path): def test_description_length_limit_is_1024_characters(tmp_path): - _skill(tmp_path / "skills", "at-limit", name="at-limit", description="d" * 1024) - _skill(tmp_path / "skills", "over-limit", name="over-limit", description="d" * 1025) + # "Use for " keeps a when-clause in the description, so only the length is in question. + _skill(tmp_path / "skills", "at-limit", name="at-limit", description="Use for " + "d" * 1016) + _skill(tmp_path / "skills", "over-limit", name="over-limit", description="Use for " + "d" * 1017) reviews = _reviews_by_name(tmp_path) assert reviews["at-limit"].state == "ok" @@ -114,7 +115,8 @@ def test_different_skill_md_with_the_same_name_is_not_a_duplicate(tmp_path): reviews = review_skills(read_skill_manifest(tmp_path)) assert all(r.duplicate_of is None for r in reviews) - assert all(r.state == "ok" for r in reviews) + # Not a duplicate, but a mirror that has drifted: both copies say so. + assert [[f.code for f in r.findings] for r in reviews] == [["mirror-drift"], ["mirror-drift"]] def test_disable_model_invocation_is_reported_as_a_flag_not_a_finding(tmp_path): @@ -491,3 +493,496 @@ def load(path): assert states["a"] == ("ok", None) assert states["b"][0] == "error" and "Permission denied" in states["b"][1] assert code == 1 + + + +def _codes(review) -> list[str]: + return [f.code for f in review.findings] + + +def test_relative_link_to_a_file_that_is_not_there_is_a_warning(tmp_path): + _skill(tmp_path / "skills", "guided", name="guided", description="Use for guided work.", + body="See [the guide](references/guide.md).\n") + bundled = _skill( + tmp_path / "skills", + "bundled", + name="bundled", + description="Use for bundled work.", + body="Details in [the guide](references/guide.md).\n", + ) + (bundled / "references").mkdir() + (bundled / "references" / "guide.md").write_text("# G\n", encoding="utf-8") + + reviews = _reviews_by_name(tmp_path) + assert reviews["guided"].state == "warn" + assert _codes(reviews["guided"]) == ["link-missing"] + assert reviews["bundled"].state == "ok" + + +def test_link_that_exists_from_the_catalog_root_says_the_base_is_wrong(tmp_path): + (tmp_path / "scripts").mkdir() + (tmp_path / "scripts" / "run.py").write_text("print(1)\n", encoding="utf-8") + _skill(tmp_path / "skills", "runner", name="runner", body="Run [it](scripts/run.py).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + [finding] = review.findings + assert finding.code == "link-missing" + assert "the base is wrong" in finding.message + + +def test_links_in_code_fences_urls_and_anchors_are_not_checked(tmp_path): + body = ( + "Read [the site](https://example.com/docs) or [below](#usage).\n" + "```markdown\n" + "See [an example](references/missing.md).\n" + "```\n" + "~~~\n" + "[also](scripts/missing.sh)\n" + "~~~\n" + ) + _skill(tmp_path / "skills", "clean", name="clean", body=body) + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "ok" + assert review.findings == [] + + +def test_skill_dir_variable_resolves_inside_the_skill(tmp_path): + skill = _skill(tmp_path / "skills", "vars", name="vars", + body="[a](${CLAUDE_SKILL_DIR}/references/a.md) [b](${CLAUDE_SKILL_DIR}/b.md)\n") + (skill / "references").mkdir() + (skill / "references" / "a.md").write_text("# A\n", encoding="utf-8") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["link-missing"] + assert "'${CLAUDE_SKILL_DIR}/b.md'" in review.findings[0].message + + +def test_link_out_of_the_skill_dir_to_an_existing_file_is_a_warning(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "reach", name="reach", body="See [notes](../../shared/notes.md).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-escapes-skill"] + + +def test_link_out_of_the_skill_dir_to_a_missing_file_is_a_warning(tmp_path): + _skill(tmp_path / "skills", "reach", name="reach", body="See [notes](../../nowhere.md).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-missing"] + + +def test_dotdot_code_span_to_an_existing_file_outside_the_skill_is_a_warning(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill( + tmp_path / "skills", + "reach", + name="reach", + body="Keep `../../shared/notes.md` next to it. Edit `docs/missing.md` in your project.\n", + ) + + [review] = review_skills(read_skill_manifest(tmp_path)) + # The code span naming a missing project path is not checked: only the escape is reported. + assert review.state == "warn" + assert _codes(review) == ["path-escapes-skill"] + + +def test_description_without_a_when_clause_is_a_warning(tmp_path): + _skill( + tmp_path / "skills", "plain", name="plain", description="Builds release reports from logs." + ) + _skill(tmp_path / "skills", "said", name="said", + description="Summarises deploy outcomes. Use when a release is cut.") + + reviews = _reviews_by_name(tmp_path) + assert reviews["plain"].state == "warn" + assert _codes(reviews["plain"]) == ["description-no-when"] + assert reviews["said"].state == "ok" + + +def test_hidden_skill_is_exempt_from_the_description_checks(tmp_path): + _skill(tmp_path / "skills", "hidden", name="hidden", description="Builds release reports.", + extra="disable-model-invocation: true\n") + _skill(tmp_path / "skills", "twin", name="twin", description="Builds release reports.", + extra="disable-model-invocation: true\n") + + reviews = _reviews_by_name(tmp_path) + assert reviews["hidden"].state == "ok" + assert reviews["twin"].state == "ok" + + +def test_near_duplicate_descriptions_warn_on_both_skills(tmp_path): + _skill(tmp_path / "skills", "style-check", name="style-check", + description="Use when reviewing Python pull requests for style and bugs.") + _skill(tmp_path / "skills", "style-lint", name="style-lint", + description="Use when reviewing Python pull requests for style and typing.") + + reviews = _reviews_by_name(tmp_path) + assert _codes(reviews["style-check"]) == ["description-near-duplicate"] + assert _codes(reviews["style-lint"]) == ["description-near-duplicate"] + assert "skills/style-lint" in reviews["style-check"].findings[0].message + + +def test_unrelated_descriptions_are_not_near_duplicates(tmp_path): + _skill(tmp_path / "skills", "deploy", name="deploy", + description="Use when deploying containers to Kubernetes clusters.") + _skill(tmp_path / "skills", "notes", name="notes", + description="Use when writing release notes for a changelog.") + + reviews = review_skills(read_skill_manifest(tmp_path)) + assert all(r.state == "ok" for r in reviews) + + +def test_identical_and_same_named_pairs_get_no_near_duplicate_warning(tmp_path): + same = "Use when reviewing Python pull requests for style and bugs." + _skill(tmp_path / "skills", "copy-a", name="copy", description=same) + _skill(tmp_path / "skills", "copy-b", name="copy", description=same) + other = "Builds Kubernetes container deployment manifests." + _skill(tmp_path / "plugins", "x", name="x", description=other, body="# Old\n") + _skill(tmp_path / "skills", "x", name="x", description=other, body="# New\n") + + reviews = review_skills(read_skill_manifest(tmp_path)) + assert len(reviews) == 4 + assert all("description-near-duplicate" not in _codes(r) for r in reviews) + + +def _words(prefix: str, count: int) -> str: + return " ".join(f"{prefix}{c}" for c in "abcdefghijklmnopqrstuvwxyz"[:count]) + + +@pytest.mark.parametrize( + ("shared", "own_a", "own_b", "warns"), + [(2, 1, 1, True), (24, 13, 12, False)], # 2/4 = 0.5 exactly; 24/49 just below it +) +def test_near_duplicate_threshold_is_half_the_words(tmp_path, shared, own_a, own_b, warns): + common = _words("zq", shared) + _skill(tmp_path / "skills", "one", name="one", + description=f"{common} {_words('xa', own_a)}") + _skill(tmp_path / "skills", "two", name="two", + description=f"{common} {_words('xb', own_b)}") + + reviews = review_skills(read_skill_manifest(tmp_path)) + assert ("description-near-duplicate" in _codes(reviews[0])) is warns + assert ("description-near-duplicate" in _codes(reviews[1])) is warns + + +def test_drifted_copies_of_one_skill_warn_on_both_and_name_each_other(tmp_path): + _skill(tmp_path / "plugins", "x", name="x", description="Use for x.", body="# Old\n") + _skill(tmp_path / "skills", "x", name="x", description="Use for x.", body="# New\n") + + reviews = review_skills(read_skill_manifest(tmp_path)) + assert all(r.duplicate_of is None for r in reviews) + assert [_codes(r) for r in reviews] == [["mirror-drift"], ["mirror-drift"]] + assert "differs from plugins/x" in reviews[1].findings[0].message + assert "differs from skills/x" in reviews[0].findings[0].message + + +def test_identical_copies_are_a_duplicate_and_not_drift(tmp_path): + _skill(tmp_path / "plugins", "x", name="x", description="Use for x.") + _skill(tmp_path / "skills", "x", name="x", description="Use for x.") + + reviews = review_skills(read_skill_manifest(tmp_path)) + assert [_codes(r) for r in reviews if r.duplicate_of is not None] == [["duplicate"]] + assert all("mirror-drift" not in _codes(r) for r in reviews) + + +def test_new_warnings_do_not_fail_the_scan_even_under_strict(tmp_path, capsys): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "reach", name="reach", description="Builds reports.", + body="See [notes](../../shared/notes.md).\n") + report, code = _scan_json(tmp_path, capsys, strict=True) + assert code == 0 + assert report["skills"][0]["state"] == "warn" + assert {f["code"] for f in report["skills"][0]["findings"]} == { + "description-no-when", "link-escapes-skill"} + + _skill(tmp_path / "skills", "example", name="example", description="Builds reports.", + body="Format: [Title](URL)\n") + report, code = _scan_json(tmp_path, capsys, strict=True) + assert code == 0 + states = {row["name"]: row["state"] for row in report["skills"]} + assert states == {"reach": "warn", "example": "warn"} + + +def test_the_scan_sees_a_mirror_that_discover_skills_leaves_out(tmp_path): + # discover_skills counts a byte-identical mirror once; the catalog review must still see both. + _skill(tmp_path / "plugins", "x", name="x", description="Use for x.") + _skill(tmp_path / "skills", "x", name="x", description="Use for x.") + assert [s.path.parent.relative_to(tmp_path).as_posix() for s in discover_skills(tmp_path)] == [ + "plugins/x" + ] + + reviews = review_skills(read_skill_manifest(tmp_path)) + assert [r.duplicate_of for r in reviews] == [None, "plugins/x"] + + +def test_a_link_target_the_os_refuses_is_a_missing_link_not_a_crash(tmp_path): + # A name over 255 bytes in one path part: Path.exists() raises on Python 3.11 to 3.13. + too_long = "a" * 300 + _skill(tmp_path / "skills", "healthy", name="healthy", description="Use for healthy.", + body="Run [it](references/a.md).\n") + _skill(tmp_path / "skills", "longname", name="longname", description="Use for longname.", + body=f"See [g]({too_long}.md).\n") + + reviews = _reviews_by_name(tmp_path) + assert reviews["healthy"].state == "warn" + assert _codes(reviews["longname"]) == ["link-missing"] + + +@pytest.mark.parametrize( + "destination", + ["tel:+15551234567", "urn:isbn:0451450523", "MAILTO:someone@example.com", "data:text/plain,x"], +) +def test_uri_scheme_destinations_are_not_files(tmp_path, destination): + _skill(tmp_path / "skills", "uris", name="uris", description="Use for uris.", + body=f"Reach us at [here]({destination}).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "ok" + assert review.findings == [] + + +def test_reference_definition_to_a_missing_file_is_a_warning(tmp_path): + _skill(tmp_path / "skills", "refs", name="refs", description="Use for refs.", + body="See [the guide][g] for details.\n\n[g]: references/missing.md\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-missing"] + + +def test_reference_definition_out_of_the_skill_dir_is_a_warning(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "reach", name="reach", description="Use for reach.", + body="See [the notes][n].\n\n[n]: ../../shared/notes.md \"Notes\"\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-escapes-skill"] + + +def test_reference_definitions_that_name_no_file_are_not_checked(tmp_path): + body = ( + "A footnote[^1] and [a site][web].\n\n" + "[^1]: Footnote text, not a file.\n" + "[web]: https://example.com/docs\n" + "[anchor]: #usage\n" + "```markdown\n" + "[example]: references/missing.md\n" + "```\n" + ) + _skill(tmp_path / "skills", "clean", name="clean", description="Use for clean.", body=body) + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "ok" + assert review.findings == [] + + +def test_angle_bracket_destination_to_a_missing_file_is_a_warning(tmp_path): + _skill(tmp_path / "skills", "angle", name="angle", description="Use for angle.", + body="See [the guide]().\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-missing"] + + +def test_angle_bracket_destination_out_of_the_skill_dir_is_a_warning(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "reach", name="reach", description="Use for reach.", + body="See [notes](<../../shared/notes.md>).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-escapes-skill"] + + +def test_angle_bracket_destination_with_an_anchor_out_of_the_skill_dir_is_a_warning(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "notes.md").write_text("# Notes\n", encoding="utf-8") + _skill(tmp_path / "skills", "reach", name="reach", description="Use for reach.", + body="See [setup](<../../shared/notes.md#setup>).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-escapes-skill"] + + +def test_reference_definition_with_spaces_in_angle_brackets_to_a_missing_file_is_a_warning(tmp_path): + _skill(tmp_path / "skills", "spaced", name="spaced", description="Use for spaced.", + body="See [notes][n].\n\n[n]: \n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-missing"] + + +@pytest.mark.parametrize( + "line", + ["[Note]: read this first", "[Step 1]: Run the tests", "[--verbose]: print more"], +) +def test_prose_line_that_starts_with_a_bracketed_label_is_not_a_definition(tmp_path, line): + _skill(tmp_path / "skills", "prose", name="prose", description="Use for prose.", + body=f"Start here.\n\n{line}\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "ok" + assert review.findings == [] + + +@pytest.mark.parametrize("destination", ["run.py:42", "run.py:10-20", "scripts/run.py:42:7"]) +def test_a_file_line_link_to_a_missing_file_is_still_a_warning(tmp_path, destination): + # "run.py:42" is read as a path, not a URI, so it is checked and the missing file is flagged. + _skill(tmp_path / "skills", "fileline", name="fileline", description="Use for fileline.", + body=f"See [the entry point]({destination}).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.state == "warn" + assert _codes(review) == ["link-missing"] + + +@pytest.mark.parametrize( + "destination", ["z39.50r:example.org/database", "x.y+z:records/item"], +) +def test_dotted_uri_scheme_destinations_are_not_files(tmp_path, destination): + _skill(tmp_path / "skills", "dotted", name="dotted", description="Use for dotted.", + body=f"Look up [the record]({destination}).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_skill_dir_variable_with_another_variable_is_not_read_as_a_file(tmp_path): + _skill(tmp_path / "skills", "tpl", name="tpl", description="Use for tpl.", + body="Run [it](${CLAUDE_SKILL_DIR}/scripts/${SCRIPT_NAME}.py).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_root_skill_and_a_nested_skill_named_like_the_checkout_are_not_mirror_drift(tmp_path): + # The root SKILL.md is named by the checkout's directory, so it is no copy of skills/tools. + root = tmp_path / "tools" + _root_skill(root, "name: root-skill\ndescription: Use for root work.\n") + _skill(root, "skills/tools", name="tools", description="Use for tool work.") + + reviews = _reviews_by_name(root) + assert _codes(reviews["root-skill"]) == [] + assert _codes(reviews["tools"]) == [] + + +def test_root_skills_of_two_checkouts_are_near_duplicates_not_mirror_drift(tmp_path): + # Both root skills reached as "sub/.." have no directory name, so they never mirror each other. + entries = [] + for checkout, ending in (("a/repo", "bugs"), ("b/repo", "typing")): + root = tmp_path / checkout + root.mkdir(parents=True) + _root_skill( + root, f"name: r{checkout[0]}\ndescription: Use when reviewing Python pull " + f"requests for style and {ending}.\n") + (root / "sub").mkdir() + (root / "skills.yaml").write_text( + f"skills:\n - name: r{checkout[0]}\n catalog_path: sub/..\n", encoding="utf-8") + entries.extend(read_skill_manifest(root)) + + reviews = review_skills(entries) + assert [_codes(r) for r in reviews] == [["description-near-duplicate"]] * 2 + for review, other in zip(reviews, ("b/repo", "a/repo"), strict=True): + assert str((tmp_path / other).as_posix()) in review.findings[0].message + + +def test_mirror_drift_names_the_checkout_of_the_other_copy(tmp_path): + entries = [] + for checkout, text in (("a", "one"), ("b", "two")): + root = tmp_path / checkout + _skill(root, "same", name="same", description=f"Use for {text}.") + entries.extend(read_skill_manifest(root)) + + reviews = review_skills(entries) + for review, other in zip(reviews, ("b", "a"), strict=True): + assert (tmp_path / other).as_posix() in review.findings[0].message + + +def test_near_duplicate_check_reads_non_ascii_description_words(tmp_path): + shared = "Use when проверка исходного кода необходима." + _skill(tmp_path / "skills", "code-check", name="code-check", description=shared) + _skill(tmp_path / "skills", "code-audit", name="code-audit", description=shared) + + reviews = _reviews_by_name(tmp_path) + assert _codes(reviews["code-check"]) == ["description-near-duplicate"] + assert _codes(reviews["code-audit"]) == ["description-near-duplicate"] + + +def test_a_file_symbol_link_to_a_missing_file_is_still_a_warning(tmp_path): + _skill(tmp_path / "skills", "symbol", name="symbol", description="Use for symbol.", + body="See [the entry point](run.py:main).\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["link-missing"] + + +@pytest.mark.parametrize( + "description", + [ + "Run database migrations when deploying a new release.", + "Diagnose failures when debugging production incidents.", + "Format commit messages when the user asks for a commit.", + ], +) +def test_a_general_when_clause_is_not_a_description_warning(tmp_path, description): + _skill(tmp_path / "skills", "clause", name="clause", description=description) + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == [] + + +@pytest.mark.parametrize("body", [ + "See [the entry point](run.py:42).\n", + "See [the entry point](run.py:main).\n", + "See [the entry point](run.py:10-20).\n", + "See [the entry point](scripts/run.py:42:7).\n", +]) +def test_an_existing_file_with_a_line_or_symbol_suffix_is_not_a_missing_link(tmp_path, body): + skill_dir = _skill(tmp_path / "skills", "bundled-run", name="bundled-run", + description="Use for bundled run.", body=body) + (skill_dir / "run.py").write_text("print('x')\n", encoding="utf-8") + (skill_dir / "scripts").mkdir() + (skill_dir / "scripts" / "run.py").write_text("print('x')\n", encoding="utf-8") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert review.findings == [] + + +def test_an_escaping_link_with_a_line_suffix_is_named_as_escaping(tmp_path): + (tmp_path / "shared").mkdir() + (tmp_path / "shared" / "run.py").write_text("print('x')\n", encoding="utf-8") + _skill(tmp_path / "skills", "reach-run", name="reach-run", description="Use for reach run.", + body="See [the entry point](../../shared/run.py:42).\n" + "Also `../../shared/run.py:main`.\n") + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert sorted(_codes(review)) == ["link-escapes-skill", "path-escapes-skill"] + + +@pytest.mark.parametrize( + "description", + [ + "Use formatting tools to normalize logs.", + "Use iframe helpers to inspect embedded pages.", + "Use ifs-free helpers to rewrite conditions.", + ], +) +def test_a_use_phrase_that_only_starts_like_a_trigger_is_still_a_description_warning(tmp_path, description): + _skill(tmp_path / "skills", "prefix-only", name="prefix-only", description=description) + + [review] = review_skills(read_skill_manifest(tmp_path)) + assert _codes(review) == ["description-no-when"] diff --git a/dokimi/tests/test_reference_checks.py b/dokimi/tests/test_reference_checks.py index 983b1a09..346b4d67 100644 --- a/dokimi/tests/test_reference_checks.py +++ b/dokimi/tests/test_reference_checks.py @@ -14,10 +14,13 @@ import json import time +from types import SimpleNamespace import pytest +import regex from dokimi.artifacts import save_artifact +from dokimi.eval import reference_checks from dokimi.eval.checklist import ASK_COVERED from dokimi.eval.reference_checks import ( REGEX_TIMEOUT, chosen_letters, grade, match_text, parse_json, validate_grading, @@ -189,11 +192,26 @@ def test_an_all_mode_regex_is_not_settled_by_its_first_match(): assert _one({"type": "regex", "values": ["a", "zzz"], "mode": "all"}, "a").passed is False -def test_the_regex_limit_is_shared_by_every_pattern_of_one_check(): - started = time.monotonic() - g = _one({"type": "regex", "values": ["(a|aa)+$", "(a|aa)+$"]}, "a" * 40 + "!") +def test_the_regex_limit_is_shared_by_every_pattern_of_one_check(monkeypatch): + # One budget per check: a fake clock charges each search 0.6x the limit, so the second + # pattern has only 0.4x left and times out (a per-pattern budget would let it finish). + now = [0.0] + monkeypatch.setattr(reference_checks, "time", SimpleNamespace(monotonic=lambda: now[0])) + searched = [] + + def search(pattern, string, flags=0, timeout=None): + searched.append(pattern) + cost = 0.6 * REGEX_TIMEOUT + if cost > timeout: + now[0] += timeout + raise TimeoutError("regex search ran past its timeout") + now[0] += cost + return object() # a match + + monkeypatch.setattr(regex, "search", search) + g = _one({"type": "regex", "values": ["p1", "p2"], "mode": "all"}, "answer") assert g.checks[0].passed is None - assert time.monotonic() - started < 1.5 * REGEX_TIMEOUT + assert searched == ["p1", "p2"] def test_a_question_with_one_failing_check_fails_without_a_threshold():