diff --git a/assessor/assessment/assemble.py b/assessor/assessment/assemble.py index 3deb9dd..e4cd5ed 100644 --- a/assessor/assessment/assemble.py +++ b/assessor/assessment/assemble.py @@ -3,6 +3,7 @@ from . import content as content_mod from . import coords, license as license_mod, probes, purpose, subject, versions +from . import record as record_mod from .shapes import fact, unknown from .fingerprint import build_payload, compute_fingerprint from .registry import (TYPE_MODULES, TIER_CONF, REGISTRY_VERSION, FULL_TAXONOMY, @@ -52,6 +53,11 @@ def build(repo_url: str, content: dict[str, str], commit_sha: str | None, if subdir: content = subject.filter_content_to_subdir(content, subdir) paths = subject.filter_paths_to_subdir(paths, subdir) + # `declared` reads `content` AFTER the subdir filter above, so a subdir subject + # reads its own re-rooted asset-record.json (record.py's parse_record docstring). + # Deliberately NOT threaded into build_payload/compute_fingerprint below — it is + # served-only, never hashed (spec §7 trust rule: nothing in record.py classifies). + declared = record_mod.declared_block(content) b = bucket_b or {} meta = {"description": b.get("description"), "topics": [t.lower() for t in (b.get("topics") or []) if isinstance(t, str)], @@ -237,7 +243,7 @@ def build(repo_url: str, content: dict[str, str], commit_sha: str | None, "license": {"spdx": lic["spdx"], "caveat": lic["caveat"]}, "coordinates": coordinates, "versions": versions.build(releases, commit_sha), "risk": risk_flags, "known_unknowns": known, "topics": topics, - "coverage_probes": coverage_probes, + "coverage_probes": coverage_probes, "declared": declared, # Audit trail for invariant-gated promotions (Task 7): which type was promoted, # from which probe state, on which LLM citation, at which confidence. Empty for # every repo that had no citation-backed candidate — i.e. almost all of them. diff --git a/assessor/assessment/content.py b/assessor/assessment/content.py index c445b8f..124952e 100644 --- a/assessor/assessment/content.py +++ b/assessor/assessment/content.py @@ -11,6 +11,7 @@ from collections import Counter from ..vendored import _valid_slug +from .record import RECORD_BASENAME, RECORD_MAX_BYTES _PHASE1 = ("README.md", "README.rst", "package.json", "pyproject.toml", "mcp.json", "server.json", "smithery.yaml", "smithery.json", "Dockerfile", @@ -115,11 +116,12 @@ def _get(path): _SRC_MAX_FILES = 120 # was 60 _SRC_MAX_BYTES = 1_600 * 1024 # was 800 * 1024 -ALLOWLIST_VERSION = 7 # bump on every allowlist widening; acquire stores it and +ALLOWLIST_VERSION = 8 # bump on every allowlist widening; acquire stores it and # SHA-reuse requires a match so stale snapshots refetch (§4.2) # 4->5: dependency manifests in subdirs + non-Python/JS ecosystems. # 5->6: marker-driven selection + raised source caps. # 6->7: agent_run_shape v3 co-occurrence changes marker-driven selection. + # 7->8: asset-record.json budget-neutral selection (authoring MCP spec §6). _AGENT_MANIFESTS = ("agents.yaml", "tasks.yaml", "langgraph.json", "agent.json", "agent-card.json") _MANIFEST_MAX_FILES, _MANIFEST_MAX_BYTES = 8, 64 * 1024 @@ -154,6 +156,16 @@ def dep_manifest_paths(paths) -> list[str]: _NEWTYPE_MANIFESTS = ("SKILL.md", "dataset_infos.json") _SKILL_MAX_FILES = 40 +# asset-record.json (spec 2026-08-09-authoring-mcp §7 channel 4): selected OUTSIDE +# every shared budget — never counts toward total/mtotal/src_files, never sets +# `capped` (which asserts SOURCE truncation), never displaces any other file. +# Oversized bodies are skipped (the reader would reject them anyway); overflow +# beyond the file cap is dropped silently — matches `_SKILL_MAX_FILES` (a legitimate +# multi-asset monorepo can legitimately have >20 subdir records, so the cap isn't +# about rejecting that case). The per-file RECORD_MAX_BYTES cap bounds the worst +# case at 640KiB (40 * 16KiB) of budget-neutral reads. +_RECORD_MAX_FILES = 40 + # Tool-definition trees (e.g. a TS MCP server's src/tools/*.ts, spread across many # per-tool subdirectories) and package/module entrypoints (e.g. a monorepo package's # packages//src/index.ts) routinely lose the density sort to a single denser, @@ -319,6 +331,10 @@ def select_source_paths(candidates: dict, *, skip=frozenset(), hits=()) -> tuple `_AGENT_MANIFESTS`. Without the dedup a source-file entrypoint consumes a cap slot/byte budget and manifests double-count, dropping a different tail file than REST would. + + A terminal pass then adds any asset-record.json candidates (`_RECORD_MAX_FILES`, + `RECORD_MAX_BYTES`) outside every budget above — see the comment at their + definitions for why that pass never counts toward SOURCE totals or `capped`. """ blob_paths = [p for p in candidates if p not in skip and _is_source(p)] @@ -480,4 +496,11 @@ def _lift(names): src_files += 1 total += size + record_paths = sorted( + p for p in candidates + if p.rsplit("/", 1)[-1] == RECORD_BASENAME + and p not in skip and p not in selected + and not any(x in p.lower() for x in _SRC_EXCLUDE) + and _candidate_size(candidates, p) <= RECORD_MAX_BYTES) + selected.extend(record_paths[:_RECORD_MAX_FILES]) return selected, capped diff --git a/assessor/assessment/git_fetch.py b/assessor/assessment/git_fetch.py index 6036f3e..188e444 100644 --- a/assessor/assessment/git_fetch.py +++ b/assessor/assessment/git_fetch.py @@ -24,6 +24,7 @@ from . import content from . import markers +from . import record from ..vendored import _REPO_KEY_RE, _SHA_RE _ALLOWED_HOSTS = frozenset({"github.com"}) @@ -175,13 +176,28 @@ def _read_blobs(git_dir: Path, shas: list[str], *, run_bytes: BytesRunner, timeo _MAX_MARKER_HITS = 500 +def _skip_marker_scan(path: str) -> bool: + """asset-record.json is excluded from marker scanning (spec §7 channel 2): + a record naming provider env vars or 'mcp server' must not generate + persisted marker_hits from its own text.""" + return path.rsplit("/", 1)[-1] == record.RECORD_BASENAME + + def _scan_present_blobs(bodies: dict[str, str]) -> list[dict]: """Marker hits across already-read blob bodies, in path order. Pure over its argument: - the caller decides which blobs are present and affordable to read.""" + the caller decides which blobs are present and affordable to read. + + This is the SOLE production call site of `markers.scan_text` in this module — every + scan pass in `GitContentFetcher.fetch` (manifest pre-scan, source pre-scan, and the + final re-scan over the fully-selected `files`) routes through here, so the + `_skip_marker_scan` guard below covers all of them without needing a second guard + elsewhere.""" hits: list[dict] = [] for path in sorted(bodies): if not markers.is_scannable(path): continue + if _skip_marker_scan(path): + continue for h in markers.scan_text(path, bodies[path]): hits.append(h) if len(hits) >= _MAX_MARKER_HITS: @@ -393,10 +409,10 @@ def _ls_tree(self, git_dir: Path, sha: str) -> tuple[list[tuple[str, str, str]], ) entries: list[tuple[str, str, str]] = [] capped = False - for record in out.split("\0"): - if not record: + for entry in out.split("\0"): + if not entry: continue - meta, sep, path = record.partition("\t") + meta, sep, path = entry.partition("\t") if not sep: continue mode, _type, blob_sha = meta.split(" ") diff --git a/assessor/assessment/record.py b/assessor/assessment/record.py new file mode 100644 index 0000000..a6d8003 --- /dev/null +++ b/assessor/assessment/record.py @@ -0,0 +1,115 @@ +"""Tolerant reading of asset-record.json — the authoring-time foundational record +(spec 2026-08-09-authoring-mcp-design.md §4, §9 D1). + +Malformed = absent (never raises, never fails an assessment): a record-level +defect (bad JSON, wrong/missing record_version, oversized body, non-dict top +level) returns None; a FIELD-level defect drops that field and keeps the rest. + +TRUST RULE (spec §7): nothing in this module feeds classification. declared_block +produces fact envelopes for the served `declared` section only.""" +from __future__ import annotations + +import json + +from .shapes import fact + +RECORD_BASENAME = "asset-record.json" +RECORD_MAX_BYTES = 16 * 1024 +_MAX_STR = 200 +_MAX_DEP = 100 +_MAX_DEPS = 50 +_SOURCE = "declared: asset-record.json" + + +def _clean_str(v) -> str | None: + return v if isinstance(v, str) and 0 < len(v) <= _MAX_STR else None + + +def _clean_model_access(v) -> dict | None: + if not isinstance(v, dict): + return None + mode = v.get("mode") + if mode not in ("pinned", "byo"): + return None + provider, model = _clean_str(v.get("provider")), _clean_str(v.get("model")) + if mode == "pinned" and not (provider and model): + return None # a pin that names no model asserts nothing + if mode == "byo": + provider = model = None + return {"mode": mode, "provider": provider, "model": model} + + +def _clean_technologies(v) -> dict | None: + if not isinstance(v, dict): + return None + deps_in = v.get("dependencies") + kept = [d for d in deps_in if isinstance(d, str) and 0 < len(d) <= _MAX_DEP] \ + if isinstance(deps_in, list) else [] + # A list that survives filtering to empty (every entry rejected, or the list was + # empty to start) collapses to None — never an affirmative empty [] that would + # read as a declared "zero dependencies" fact. + deps = (kept[:_MAX_DEPS] or None) if isinstance(deps_in, list) else None + out = {"language": _clean_str(v.get("language")), + "framework": _clean_str(v.get("framework")), + "runtime": _clean_str(v.get("runtime")), + "dependencies": deps} + return out if any(x is not None for x in out.values()) else None + + +def _clean_source_repo(v) -> dict | None: + if not isinstance(v, dict): + return None + host, owner, name = (_clean_str(v.get(k)) for k in ("host", "owner", "name")) + return {"host": host, "owner": owner, "name": name} if host and owner and name else None + + +def parse_record(content: dict[str, str]) -> dict | None: + """The validated record, or None. Reads content[RECORD_BASENAME] — after + filter_content_to_subdir a subdir's record re-roots to exactly this key, so + one lookup serves both whole-repo and subdir subjects.""" + body = content.get(RECORD_BASENAME) + if not isinstance(body, str) or len(body.encode("utf-8")) > RECORD_MAX_BYTES: + return None + try: + raw = json.loads(body) + except (ValueError, TypeError, RecursionError): + # RecursionError can occur on deeply-nested JSON; treat as malformed + return None + if not isinstance(raw, dict): + return None + rv = raw.get("record_version") + # Exclude booleans: True == 1 but is not a valid version + if not (isinstance(rv, int) and not isinstance(rv, bool) and rv == 1): + return None + conf = raw.get("confirmation") + return { + "record_version": 1, + "created_by": _clean_str(raw.get("created_by")), + "created_at": _clean_str(raw.get("created_at")), + "maintained_by": _clean_str(raw.get("maintained_by")), + "source_repo": _clean_source_repo(raw.get("source_repo")), + "technologies": _clean_technologies(raw.get("technologies")), + "model_access": _clean_model_access(raw.get("model_access")), + "confirmation": conf if isinstance(conf, dict) else None, + } + + +def _facts(section: str, values: dict) -> dict: + return {k: fact(v, _SOURCE, [{"path": RECORD_BASENAME, "marker": f"{section}.{k}"}]) + for k, v in values.items() if v is not None} + + +def declared_block(content: dict[str, str]) -> dict | None: + """The `declared` section of the assessment payload: technologies + + model_access as fact envelopes (spec §4 — these two land on the ASSESSMENT; + created_by/maintained_by/created_at land on creation-info via repo_meta and + are deliberately NOT emitted here).""" + rec = parse_record(content) + if rec is None: + return None + out: dict = {} + if rec["technologies"]: + out["technologies"] = _facts("technologies", rec["technologies"]) + if rec["model_access"]: + out["model_access"] = _facts("model_access", rec["model_access"]) + return out or None diff --git a/assessor/ports/source.py b/assessor/ports/source.py index 32ac0ee..f81b819 100644 --- a/assessor/ports/source.py +++ b/assessor/ports/source.py @@ -10,6 +10,7 @@ from urllib.parse import urlsplit from ..assessment import content as content_mod +from ..assessment import record from ..errors import RepoGone from ..vendored import _valid_slug @@ -158,6 +159,18 @@ def acquire(self, repo_url: str, *, prior: Prior | None) -> AcquireResult: # equivalent and keeps the on-disk cache path fixed per repo. repo_key = hashlib.sha256(f"{owner}/{name}".encode()).hexdigest() files, paths, capped, hits = self.fetcher.fetch(clone_url, repo_key, sha) + # declared_* keys land ONLY on this branch: `unchanged` fetches no + # files, so the record is unreadable there, and emitting None keys on + # that branch would look identical to "record removed" to CodeRoot's + # executor (which preserves prior DB values on `unchanged`) — see + # Task 5 brief for the full rationale. + rec = record.parse_record(files) + repo_meta = dict(repo_meta) + repo_meta.update({ + "declared_created_by": rec["created_by"] if rec else None, + "declared_maintained_by": rec["maintained_by"] if rec else None, + "declared_created_at": rec["created_at"] if rec else None, + }) return {"status": "acquired", "commit_sha": sha, "metadata": meta, "repo_meta": repo_meta, "allowlist_version": alv, "snapshot": {"commit_sha": sha, "metadata": meta, diff --git a/tests/assessment/test_assessment_content.py b/tests/assessment/test_assessment_content.py index 6dc004a..13a639a 100644 --- a/tests/assessment/test_assessment_content.py +++ b/tests/assessment/test_assessment_content.py @@ -165,8 +165,8 @@ def test_phase1_fetches_requirements_and_langgraph_json(): assert "requirements.txt" in out["files"] and "langgraph.json" in out["files"] -def test_allowlist_version_is_7(): - assert ALLOWLIST_VERSION == 7 # 6->7: agent_run_shape v3 co-occurrence changes marker-driven selection +def test_allowlist_version_is_8(): + assert ALLOWLIST_VERSION == 8 # 7->8: asset-record.json budget-neutral selection (authoring MCP spec §6). def test_select_source_paths_selects_skill_md_bodies_independent_of_is_source(): @@ -256,11 +256,11 @@ def test_non_agent_hit_does_not_displace_a_tool_reservation(): assert chosen.index("src/tools/a.ts") < chosen.index("cfg/settings.py") -def test_hit_paths_that_are_not_candidates_are_ignored_and_allowlist_is_7(): +def test_hit_paths_that_are_not_candidates_are_ignored_and_allowlist_is_8(): cands = {"lib/a.ts": 10} chosen, _ = select_source_paths(cands, hits=[{**_SHAPE, "path": "gone/x.py"}]) assert chosen == ["lib/a.ts"] - assert ALLOWLIST_VERSION == 7 + assert ALLOWLIST_VERSION == 8 def test_agent_lift_is_capped_so_the_tool_reservation_always_survives(): diff --git a/tests/test_app_http.py b/tests/test_app_http.py index 7e1666d..1b439ff 100644 --- a/tests/test_app_http.py +++ b/tests/test_app_http.py @@ -113,7 +113,7 @@ def test_version_endpoint_reports_all_three_versions(): assert set(r.json()) == {"registry_version", "allowlist_version", "marker_vocab_version"} assert r.json()["registry_version"] == 10 - assert r.json()["allowlist_version"] == 7 + assert r.json()["allowlist_version"] == 8 def test_healthz_needs_no_auth(): @@ -174,7 +174,7 @@ def get_subject(self, repo_id, subdir=""): return {"commit_sha": "abc123", "description": None, "homepage": None, "topics": [], "license_spdx": None, "tree_paths": list(_MCP), "content_paths": list(_MCP), "tree_capped": False, "marker_hits": [], - "source_coverage_capped": False, "allowlist_version": 7} + "source_coverage_capped": False, "allowlist_version": 8} def read_files(self, repo_id, commit_sha, paths): return {"files": dict(_MCP), "missing": []} diff --git a/tests/test_handlers_assess.py b/tests/test_handlers_assess.py index f8d8d75..3ee41c1 100644 --- a/tests/test_handlers_assess.py +++ b/tests/test_handlers_assess.py @@ -52,12 +52,13 @@ def test_returns_the_seven_top_level_record_keys(): "llm_used", "assessment"} -def test_assessment_payload_has_exactly_the_fifteen_keys(): +def test_assessment_payload_has_exactly_the_sixteen_keys(): r = assess_handler(_Source(_MCP), NullCache(), _S, _SUBJECT) assert set(r["assessment"]) == { "classification", "purpose", "compositions", "composition", "license", "coordinates", "versions", "risk", "known_unknowns", "topics", - "coverage_probes", "promoted_types", "subdir", "asset_id", "source_url"} + "coverage_probes", "promoted_types", "subdir", "asset_id", "source_url", + "declared"} def test_interop_is_absent(): diff --git a/tests/test_ports_source.py b/tests/test_ports_source.py index ace7a8e..2460fde 100644 --- a/tests/test_ports_source.py +++ b/tests/test_ports_source.py @@ -55,7 +55,7 @@ def test_matching_prior_performs_no_clone(monkeypatch): f = _Fetcher() src = _direct(_Http(sha="abc123"), f, monkeypatch) r = src.acquire("https://github.com/o/n", - prior={"commit_sha": "abc123", "allowlist_version": 7}) + prior={"commit_sha": "abc123", "allowlist_version": 8}) assert r["status"] == "unchanged" assert f.calls == 0 @@ -75,7 +75,7 @@ def test_unchanged_still_returns_refreshed_metadata(monkeypatch): http = _Http(sha="abc123", repo_obj={"description": "fresh", "topics": ["mcp"]}) src = _direct(http, _Fetcher(), monkeypatch) r = src.acquire("https://github.com/o/n", - prior={"commit_sha": "abc123", "allowlist_version": 7}) + prior={"commit_sha": "abc123", "allowlist_version": 8}) assert r["status"] == "unchanged" assert r["metadata"]["description"] == "fresh" @@ -196,7 +196,12 @@ def test_noassertion_license_is_normalized_to_none(): def test_acquire_result_carries_repo_meta_on_both_branches(monkeypatch): """`unchanged` (SHA reuse) must carry refreshed metadata too — spec §5.1: - Bucket B is refreshed on every run including the reuse path.""" + Bucket B is refreshed on every run including the reuse path. + + The acquired branch additionally carries the three declared_* keys (Task + 5) that `unchanged` deliberately omits — see test_record_repo_meta.py for + the dedicated coverage of that asymmetry; this test only pins that the + baseline repo_meta columns keep showing up on both branches.""" repo_obj = { "default_branch": "main", "description": "d", "homepage": "h", "topics": ["a"], "license": {"spdx_id": "MIT"}, @@ -211,14 +216,15 @@ def test_acquire_result_carries_repo_meta_on_both_branches(monkeypatch): src = _direct(http, _Fetcher(), monkeypatch) r = src.acquire("https://github.com/o/n", prior=None) assert r["status"] == "acquired" - assert set(r["repo_meta"]) == _EXPECTED_REPO_META_KEYS + assert set(r["repo_meta"]) == _EXPECTED_REPO_META_KEYS | { + "declared_created_by", "declared_maintained_by", "declared_created_at"} assert r["repo_meta"]["repo_owner_login"] == "o" # unchanged branch: matching prior short-circuits the clone, metadata # (including repo_meta) is still refreshed from resolve_head. src2 = _direct(http, _Fetcher(), monkeypatch) r2 = src2.acquire("https://github.com/o/n", - prior={"commit_sha": "abc123", "allowlist_version": 7}) + prior={"commit_sha": "abc123", "allowlist_version": 8}) assert r2["status"] == "unchanged" assert set(r2["repo_meta"]) == _EXPECTED_REPO_META_KEYS assert r2["repo_meta"]["repo_owner_login"] == "o" diff --git a/tests/test_record.py b/tests/test_record.py new file mode 100644 index 0000000..6382541 --- /dev/null +++ b/tests/test_record.py @@ -0,0 +1,156 @@ +"""Tolerant reading of asset-record.json (spec §4, §9 D1). + +A malformed record is an ABSENT record (Global Constraint 6): this module +never raises on bad input and never fails an assessment.""" +import json + +from assessor.assessment.record import (RECORD_BASENAME, RECORD_MAX_BYTES, + declared_block, parse_record) + +GOOD = { + "record_version": 1, + "created_by": "settletop-niles", + "created_at": "2026-08-09T00:00:00Z", + "source_repo": {"host": "github.com", "owner": "SettleTop-Inc", "name": "example"}, + "maintained_by": "SettleTop-Inc", + "technologies": {"language": "python", "framework": "mcp", + "runtime": "python>=3.11", "dependencies": ["mcp", "httpx"]}, + "model_access": {"mode": "byo", "provider": None, "model": None}, + "confirmation": {"mode": "elicitation", + "confirmed": ["created_by", "maintained_by", "model_access.mode"], + "complete": True}, +} + + +def _content(rec) -> dict[str, str]: + return {RECORD_BASENAME: json.dumps(rec), "README.md": "hi"} + + +def test_parse_good_record(): + rec = parse_record(_content(GOOD)) + assert rec["created_by"] == "settletop-niles" + assert rec["model_access"]["mode"] == "byo" + assert rec["technologies"]["dependencies"] == ["mcp", "httpx"] + + +def test_absent_file_is_none(): + assert parse_record({"README.md": "hi"}) is None + + +def test_malformed_json_is_none(): + assert parse_record({RECORD_BASENAME: "{not json"}) is None + + +def test_wrong_record_version_is_none(): + bad = dict(GOOD, record_version=2) + assert parse_record(_content(bad)) is None + + +def test_missing_record_version_is_none(): + bad = {k: v for k, v in GOOD.items() if k != "record_version"} + assert parse_record(_content(bad)) is None + + +def test_non_dict_top_level_is_none(): + assert parse_record({RECORD_BASENAME: json.dumps([1, 2])}) is None + + +def test_oversized_body_is_none(): + big = dict(GOOD) + big["technologies"] = dict(GOOD["technologies"], dependencies=["x" * 90] * 50) + body = json.dumps(big) + " " * RECORD_MAX_BYTES + assert parse_record({RECORD_BASENAME: body}) is None + + +def test_pinned_mode_requires_provider_and_model(): + ok = dict(GOOD, model_access={"mode": "pinned", "provider": "anthropic", + "model": "claude-opus-5"}) + assert parse_record(_content(ok))["model_access"]["model"] == "claude-opus-5" + bad = dict(GOOD, model_access={"mode": "pinned", "provider": None, "model": None}) + # pinned-without-model is a field-level defect: the field drops, the record survives + rec = parse_record(_content(bad)) + assert rec is not None and rec.get("model_access") is None + + +def test_invalid_mode_drops_field_keeps_record(): + bad = dict(GOOD, model_access={"mode": "sometimes"}) + rec = parse_record(_content(bad)) + assert rec is not None and rec.get("model_access") is None + + +def test_overlong_strings_dropped_field_level(): + bad = dict(GOOD, created_by="x" * 201) + rec = parse_record(_content(bad)) + assert rec is not None and rec.get("created_by") is None + + +def test_dependencies_capped_at_50(): + rec = parse_record(_content(dict(GOOD, technologies=dict( + GOOD["technologies"], dependencies=[f"d{i}" for i in range(60)])))) + assert len(rec["technologies"]["dependencies"]) == 50 + + +def test_unknown_keys_ignored(): + rec = parse_record(_content(dict(GOOD, future_field={"a": 1}))) + assert rec is not None and "future_field" not in rec + + +def test_declared_block_shapes_facts(): + block = declared_block(_content(GOOD)) + assert block["technologies"]["language"]["value"] == "python" + assert block["technologies"]["language"]["source"] == "declared: asset-record.json" + assert block["model_access"]["mode"]["value"] == "byo" + # evidence cites the file + assert block["technologies"]["framework"]["evidence"] == [ + {"path": RECORD_BASENAME, "marker": "technologies.framework"}] + + +def test_declared_block_absent_when_no_record(): + assert declared_block({"README.md": "x"}) is None + + +def test_declared_block_omits_dropped_fields(): + bad = dict(GOOD, model_access={"mode": "sometimes"}) + block = declared_block(_content(bad)) + assert "model_access" not in block + assert "technologies" in block + + +def test_deeply_nested_json_returns_none(): + # Deeply-nested JSON can cause RecursionError in json.loads; must not propagate + # 8000 bytes of nested brackets, under the 16KiB cap + body = "[" * 8000 + assert parse_record({RECORD_BASENAME: body}) is None + + +def test_record_version_true_rejected(): + # Boolean True == 1 in Python, but must not pass version gate + bad = dict(GOOD, record_version=True) + assert parse_record(_content(bad)) is None + + +def test_all_dependencies_rejected_collapses_to_none(): + # Every entry over _MAX_DEP (100 chars) is rejected field-by-field; the survivor + # list is empty, not []. An empty list must NOT read as an affirmative declared + # "zero dependencies" fact -- it must collapse to None like an absent field. + bad = dict(GOOD, technologies=dict( + GOOD["technologies"], dependencies=["d" * 300, "e" * 400])) + rec = parse_record(_content(bad)) + assert rec["technologies"]["dependencies"] is None + + block = declared_block(_content(bad)) + # language/framework/runtime are still set, so the technologies block survives -- + # but it must omit the dependencies key entirely, never assert an empty list. + assert "dependencies" not in block["technologies"] + + +def test_all_dependencies_rejected_and_no_other_fields_omits_technologies(): + # When dependencies is the ONLY populated technologies key and every entry is + # rejected, the whole technologies block must be absent from parse_record and + # declared_block -- not present with every value None. + bad = dict(GOOD, technologies={"dependencies": ["d" * 300, "e" * 400]}) + rec = parse_record(_content(bad)) + assert rec["technologies"] is None + + block = declared_block(_content(bad)) + assert block is None or "technologies" not in block diff --git a/tests/test_record_declared_block.py b/tests/test_record_declared_block.py new file mode 100644 index 0000000..7a6914a --- /dev/null +++ b/tests/test_record_declared_block.py @@ -0,0 +1,66 @@ +"""The `declared` block on the served assessment (Task 6, spec §4). + +`assemble.build` computes `declared_block(content)` (record.py, Task 1) and serves +it at `assessment["declared"]` — but must NOT let it leak into `build_payload`/ +`compute_fingerprint`: the fingerprint-parity guard is the existing fingerprint +tests (test_assessment_fingerprint.py, test_assessment_recall_drift.py), which must +stay green untouched by this change. +""" +import json + +from assessor.assessment import assemble +from assessor.assessment.record import RECORD_BASENAME + +from conftest import _S + +# Copied from tests/test_record.py's GOOD (Task 1's contract) rather than imported, +# so this test file does not depend on test_record.py's module layout. +GOOD = { + "record_version": 1, + "created_by": "settletop-niles", + "created_at": "2026-08-09T00:00:00Z", + "source_repo": {"host": "github.com", "owner": "SettleTop-Inc", "name": "example"}, + "maintained_by": "SettleTop-Inc", + "technologies": {"language": "python", "framework": "mcp", + "runtime": "python>=3.11", "dependencies": ["mcp", "httpx"]}, + "model_access": {"mode": "byo", "provider": None, "model": None}, + "confirmation": {"mode": "elicitation", + "confirmed": ["created_by", "maintained_by", "model_access.mode"], + "complete": True}, +} + +# A content fixture that actually classifies (mirrors golden_mcp's "strong-dep" corpus +# entry), so the test exercises a real `assemble.build` asset path, not `not_an_asset`. +MCP_FIXTURE_CONTENT = { + "package.json": json.dumps({"dependencies": {"@modelcontextprotocol/sdk": "^1.0"}}), + "src/index.ts": 'srv.registerTool("search", {});', +} + + +def test_declared_block_served_when_record_present(): + content = {**MCP_FIXTURE_CONTENT, RECORD_BASENAME: json.dumps(GOOD)} + out = assemble.build("https://github.com/o/n", content, "sha", None, settings=_S) + assert out["assessment"]["declared"]["technologies"]["language"]["value"] == "python" + assert (out["assessment"]["declared"]["technologies"]["language"]["source"] + == "declared: asset-record.json") + + +def test_declared_none_when_absent(): + out = assemble.build("https://github.com/o/n", MCP_FIXTURE_CONTENT, "sha", None, settings=_S) + assert out["assessment"]["declared"] is None + + +def test_declared_block_served_for_subdir_subject(): + """A subdir subject's OWN asset-record.json — re-rooted by + `subject.filter_content_to_subdir` before `record.declared_block` ever reads + `content` (assemble.py's re-rooting-path comment) — still populates + `assessment["declared"]`, not just the whole-repo case above.""" + subdir = "mypkg" + content = { + f"{subdir}/package.json": json.dumps({"dependencies": {"@modelcontextprotocol/sdk": "^1.0"}}), + f"{subdir}/src/index.ts": 'srv.registerTool("search", {});', + f"{subdir}/{RECORD_BASENAME}": json.dumps(GOOD), + } + out = assemble.build("https://github.com/o/n", content, "sha", None, settings=_S, subdir=subdir) + assert out["assessment"]["declared"] is not None + assert out["assessment"]["declared"]["technologies"]["language"]["value"] == "python" diff --git a/tests/test_record_marker_exclusion.py b/tests/test_record_marker_exclusion.py new file mode 100644 index 0000000..2de4e50 --- /dev/null +++ b/tests/test_record_marker_exclusion.py @@ -0,0 +1,46 @@ +"""Spec §7 channel 2: an adversarial record generates ZERO marker hits.""" +import json + +from assessor.assessment import markers +from assessor.assessment.record import RECORD_BASENAME + +ADVERSARIAL_BODY = json.dumps({ + "record_version": 1, + "created_by": "x", + "technologies": {"framework": "mcp server ANTHROPIC_BASE_URL OPENAI_API_BASE", + "dependencies": ["langgraph", "crewai", "claude-opus-5"]}, +}) + + +def test_scan_text_on_record_body_would_hit(): + # Control: the body IS marker-bearing when scanned — proving the exclusion + # below is load-bearing, not vacuous. + hits = markers.scan_text("some/other/file.py", ADVERSARIAL_BODY) + assert hits, "adversarial body must trip markers when NOT excluded" + + +def test_record_path_is_never_scanned(): + # The exclusion is basename-keyed at the git_fetch scan sites. This test + # pins the helper those sites share (add `_skip_marker_scan(path)` or + # equivalent — adjust the import to the implementation). + from assessor.assessment.git_fetch import _skip_marker_scan + assert _skip_marker_scan(RECORD_BASENAME) is True + assert _skip_marker_scan(f"pkg/{RECORD_BASENAME}") is True + assert _skip_marker_scan("src/main.py") is False + # Near-miss basenames must NOT be swept in by a loose (substring/prefix) match — + # the exclusion is exact-basename only (Task 3 review). + assert _skip_marker_scan("asset-record.json.bak") is False + assert _skip_marker_scan("notasset-record.json") is False + + +def test_scan_present_blobs_skips_the_record(): + # End-to-end at the shared scan helper: an adversarial record body mixed in + # with an ordinary file produces hits only for the ordinary file. + from assessor.assessment.git_fetch import _scan_present_blobs + bodies = { + RECORD_BASENAME: ADVERSARIAL_BODY, + "src/main.py": "os.environ['ANTHROPIC_API_KEY']\n", + } + hits = _scan_present_blobs(bodies) + assert hits, "the ordinary file must still be scanned" + assert all(h["path"] != RECORD_BASENAME for h in hits) diff --git a/tests/test_record_repo_meta.py b/tests/test_record_repo_meta.py new file mode 100644 index 0000000..aa1cbf3 --- /dev/null +++ b/tests/test_record_repo_meta.py @@ -0,0 +1,101 @@ +# tests/test_record_repo_meta.py +"""Task 5: declared_* keys on repo_meta. + +DirectSource.acquire's `acquired` branch parses asset-record.json (via +assessor.assessment.record.parse_record) out of the freshly fetched `files` +dict and merges three declared_* keys into repo_meta. The `unchanged` branch +fetches no files, so it is deliberately left without these keys — emitting +None there would look identical to "the record was removed" and CodeRoot's +executor (a later task) relies on that distinction to preserve prior DB +values rather than nulling them out. +""" +import json + +from assessor.assessment import content as content_mod +from assessor.assessment import record +from assessor.config import Settings +from assessor.ports.source import DirectSource + +_S = Settings(assessor_api_token="x") + + +class _Http: + """Stands in for CodeRoot's HttpClient. resolve_head is the only call + DirectSource makes through it.""" + def __init__(self, sha="abc123", repo_obj=None, gone=False): + self.sha, self.repo_obj, self.gone = sha, repo_obj or {}, gone + + +class _RecordFetcher: + """Returns files containing a valid root asset-record.json.""" + def __init__(self): + self.calls = 0 + + def fetch(self, clone_url, repo_id, sha): + self.calls += 1 + body = json.dumps({ + "record_version": 1, + "created_by": "settletop-niles", + "created_at": "2026-08-09T00:00:00Z", + "maintained_by": "SettleTop-Inc", + }) + return ({record.RECORD_BASENAME: body}, (record.RECORD_BASENAME,), False, []) + + +class _NoRecordFetcher: + """Returns files with no asset-record.json at all.""" + def __init__(self): + self.calls = 0 + + def fetch(self, clone_url, repo_id, sha): + self.calls += 1 + return ({"README.md": "hi"}, ("README.md",), False, []) + + +def _direct(http, fetcher, monkeypatch): + def fake_resolve(h, owner, name): + if h.gone: + raise content_mod.RepoGone(f"{owner}/{name}") + return h.sha, h.repo_obj + + monkeypatch.setattr(content_mod, "resolve_head", fake_resolve) + return DirectSource(_S, http, fetcher) + + +def test_acquired_branch_carries_declared_keys(monkeypatch): + src = _direct(_Http(), _RecordFetcher(), monkeypatch) + result = src.acquire("https://github.com/o/n", prior=None) + assert result["status"] == "acquired" + assert result["repo_meta"]["declared_created_by"] == "settletop-niles" + assert result["repo_meta"]["declared_maintained_by"] == "SettleTop-Inc" + assert result["repo_meta"]["declared_created_at"] == "2026-08-09T00:00:00Z" + + +def test_no_record_yields_none_keys(monkeypatch): + """No asset-record.json in the fetched files: the three declared_* keys + are still present on repo_meta (the acquired branch always emits them), + just None rather than dropped.""" + src = _direct(_Http(), _NoRecordFetcher(), monkeypatch) + result = src.acquire("https://github.com/o/n", prior=None) + assert result["status"] == "acquired" + assert result["repo_meta"]["declared_created_by"] is None + assert result["repo_meta"]["declared_maintained_by"] is None + assert result["repo_meta"]["declared_created_at"] is None + + +def test_unchanged_branch_has_no_declared_keys(monkeypatch): + """prior matches sha+allowlist -> status unchanged, no files fetched, so + the record is unreadable there. The keys must be absent (not None) so + CodeRoot's executor can tell "no info this run" apart from "record + removed" and preserve prior DB values instead of nulling them out.""" + f = _RecordFetcher() + src = _direct(_Http(sha="abc123"), f, monkeypatch) + result = src.acquire( + "https://github.com/o/n", + prior={"commit_sha": "abc123", "allowlist_version": content_mod.ALLOWLIST_VERSION}, + ) + assert result["status"] == "unchanged" + assert "declared_created_by" not in result["repo_meta"] + assert "declared_maintained_by" not in result["repo_meta"] + assert "declared_created_at" not in result["repo_meta"] + assert f.calls == 0 diff --git a/tests/test_record_selection.py b/tests/test_record_selection.py new file mode 100644 index 0000000..d2e4cf5 --- /dev/null +++ b/tests/test_record_selection.py @@ -0,0 +1,52 @@ +"""Budget-neutral record selection (spec §7 channel 4): the record is always +selected, and its presence changes NO other file's selection.""" +from assessor.assessment.content import select_source_paths +from assessor.assessment.record import RECORD_BASENAME, RECORD_MAX_BYTES + + +def _candidates_exhausting_budgets() -> dict: + # 130 source files of 16KiB each: >120-file cap AND >1600KiB byte cap. + c = {f"src/m{i:03}.py": (f"sha{i}", 16 * 1024) for i in range(130)} + c["package.json"] = ("shaP", 200) # dep manifest + c["skills/a/SKILL.md"] = ("shaS", 300) # newtype manifest + return c + + +def test_record_always_selected_even_when_budgets_exhausted(): + c = _candidates_exhausting_budgets() + c[RECORD_BASENAME] = ("shaR", 2_000) + selected, capped = select_source_paths(c) + assert RECORD_BASENAME in selected + assert capped is True # capped by the SOURCE files, as before + + +def test_record_presence_changes_no_other_selection(): + c = _candidates_exhausting_budgets() + base, base_capped = select_source_paths(dict(c)) + c[RECORD_BASENAME] = ("shaR", 2_000) + with_rec, rec_capped = select_source_paths(c) + assert [p for p in with_rec if p != RECORD_BASENAME] == base + assert rec_capped == base_capped + + +def test_subdir_records_selected_up_to_cap(): + c = {"README.md": ("s0", 10)} + for i in range(45): + c[f"pkg{i}/{RECORD_BASENAME}"] = (f"s{i}", 500) + selected, capped = select_source_paths(c) + picked = [p for p in selected if p.endswith(RECORD_BASENAME)] + assert len(picked) == 40 # _RECORD_MAX_FILES + assert capped is False # record overflow NEVER claims source truncation + + +def test_oversized_record_not_selected(): + c = {"README.md": ("s0", 10), RECORD_BASENAME: ("sR", RECORD_MAX_BYTES + 1)} + selected, capped = select_source_paths(c) + assert RECORD_BASENAME not in selected + assert capped is False + + +def test_record_never_sets_capped_flag(): + c = {RECORD_BASENAME: ("sR", 2_000)} + selected, capped = select_source_paths(c) + assert selected == [RECORD_BASENAME] and capped is False diff --git a/tests/test_record_trust_rule.py b/tests/test_record_trust_rule.py new file mode 100644 index 0000000..f213274 --- /dev/null +++ b/tests/test_record_trust_rule.py @@ -0,0 +1,85 @@ +"""Spec §7 trust rule, pinned end-to-end (Task 7). + +Two structural properties, each closing a distinct attack surface Tasks 1-6 built +the machinery for but never pinned with a dedicated test: + +(A) The declared-identity suppressor (registry.py, spec §8.3) may only KEEP or + DROP a match classification already produced. `meta` (repo topics/description + — and, by the same reasoning, a maintainer-authored record) can steer which + weak matches survive, but it must never manufacture a brand-new match for a + type nothing deterministic pointed at. + +(B) An adversarial `asset-record.json` — one crafted to look like MCP/agent prose + and to smuggle framework/dependency vocabulary that WOULD trip classification + if it appeared anywhere else in the repo — changes nothing about the served + assessment except `assessment.declared` itself. This is the composite + end-to-end proof that channels 1 (classification reach) and 3 (prose/keyword + scanning) both hold; channel 2 (marker-hit scanning) is pinned separately in + test_record_marker_exclusion.py and channel 4 (budget-neutral selection) in + test_record_selection.py. +""" +import json + +from assessor.assessment import assemble +from assessor.assessment.record import RECORD_BASENAME + +from conftest import _S + + +def test_suppressor_cannot_introduce_a_type(): + """_apply_declared_identity_suppressor may keep or drop, never add: meta + declaring a type with NO match in `matches` must not create one.""" + from assessor.assessment.registry import _apply_declared_identity_suppressor + + # Real match-element shape, copied verbatim from test_assessment_registry.py's + # `_strong_skill()` fixture (asset_type/marker_tier/confidence/evidence) — + # no invented keys. + matches = [{"asset_type": "skill", "marker_tier": "strong", "confidence": 0.95, + "evidence": [{"path": "SKILL.md", "marker": "skill manifest"}]}] + # meta declares mcp_server identity strongly (both topics AND description), + # even though NO mcp_server match exists anywhere in `matches`. + meta = {"description": "an mcp server for things", "topics": ["mcp", "mcp-server"]} + kept, suppressed = _apply_declared_identity_suppressor(list(matches), meta) + assert {m["asset_type"] for m in kept} == {"skill"} + assert all(m["asset_type"] != "mcp_server" for m in kept) + assert suppressed == [] # the lone match is strong -- nothing to suppress either + + +# A content fixture that actually classifies (mirrors Task 6's MCP_FIXTURE_CONTENT in +# test_record_declared_block.py, copied rather than imported so this file does not +# depend on that file's module layout), so the test below exercises a real +# `assemble.build` asset path, not `not_an_asset`. +MCP_FIXTURE_CONTENT = { + "package.json": json.dumps({"dependencies": {"@modelcontextprotocol/sdk": "^1.0"}}), + "src/index.ts": 'srv.registerTool("search", {});', +} + + +def test_adversarial_record_changes_nothing_but_declared(): + """Spec §7 + §10: same repo with and without an adversarial record — + det classification, fingerprint, marker-relevant outputs byte-identical.""" + base = dict(MCP_FIXTURE_CONTENT) # a real fixture that classifies + adversarial = json.dumps({ + "record_version": 1, "created_by": "x", + "technologies": {"framework": "mcp server agent ANTHROPIC_BASE_URL", + "dependencies": ["langgraph", "crewai"]}, + "model_access": {"mode": "byo", "provider": None, "model": None}}) + # Trust channel 3 (prose): the adversarial body names exactly the words a + # keyword/prose scanner would key on. + assert "mcp server" in adversarial and "agent" in adversarial + + with_rec = dict(base) + with_rec[RECORD_BASENAME] = adversarial + + a = assemble.build("https://github.com/o/n", base, "sha", None, settings=_S) + b = assemble.build("https://github.com/o/n", with_rec, "sha", None, settings=_S) + + assert a["asset_types"] == b["asset_types"] + assert a["asset_type"] == b["asset_type"] + assert a["classification_confidence"] == b["classification_confidence"] + assert a["content_fingerprint"] == b["content_fingerprint"] + assert b["assessment"]["declared"] is not None # non-vacuity: the record WAS read + + a_rest = {k: v for k, v in a["assessment"].items() if k != "declared"} + b_rest = {k: v for k, v in b["assessment"].items() if k != "declared"} + assert a_rest == b_rest